From 4c78c86df46f1efe94fbefc51f3dd2b4e8db14ed Mon Sep 17 00:00:00 2001 From: Gang Ji Date: Fri, 25 Oct 2024 13:30:04 +0800 Subject: [PATCH] CP-50787 CP-51347: Support pool.sync_updates from remote_pool repo When a remote_pool type repository, which points to the enabled repository in the remote pool coordinator, is set as the enabled repository of the pool, updates can be synced from it with API pool.sync_updates. The username password of the remote pool coordinator is required as parameters for pool.sync_updates to login the remote pool. And the remote pool coordinator's host server certificate needs to be configured in the remote_pool repository, it will be used to verify the remote end when sending out username passwords and syncing updates from it. A new yum/dnf plugin "xapitoken" is introduced to set xapi token as HTTP cookie: "session_id" for each HTTP request which downloads files from the remote_pool repository. Signed-off-by: Gang Ji --- ocaml/idl/datamodel_errors.ml | 12 ++ ocaml/idl/datamodel_pool.ml | 14 ++ ocaml/xapi-cli-server/cli_frontend.ml | 2 +- ocaml/xapi-cli-server/cli_operations.ml | 3 + ocaml/xapi-consts/api_errors.ml | 8 + ocaml/xapi/helpers.ml | 21 ++- ocaml/xapi/pool_periodic_update_sync.ml | 3 +- ocaml/xapi/repository.ml | 195 +++++++++++++++++++----- ocaml/xapi/repository.mli | 2 + ocaml/xapi/repository_helpers.ml | 54 ++++--- ocaml/xapi/xapi_pool.ml | 13 +- ocaml/xapi/xapi_pool.mli | 2 + python3/dnf_plugins/accesstoken.py | 2 +- python3/dnf_plugins/ptoken.py | 2 + python3/dnf_plugins/xapitoken.py | 49 ++++++ python3/tests/test_dnf_plugins.py | 63 +++++++- scripts/Makefile | 2 + scripts/yum-plugins/accesstoken.py | 8 +- scripts/yum-plugins/ptoken.py | 2 +- scripts/yum-plugins/xapitoken.conf | 2 + scripts/yum-plugins/xapitoken.py | 48 ++++++ 21 files changed, 421 insertions(+), 86 deletions(-) create mode 100644 python3/dnf_plugins/xapitoken.py create mode 100644 scripts/yum-plugins/xapitoken.conf create mode 100644 scripts/yum-plugins/xapitoken.py diff --git a/ocaml/idl/datamodel_errors.ml b/ocaml/idl/datamodel_errors.ml index d1c3bf0ac0c..c6bc9ed01d3 100644 --- a/ocaml/idl/datamodel_errors.ml +++ b/ocaml/idl/datamodel_errors.ml @@ -1903,6 +1903,8 @@ let _ = ~doc:"The base url in the repository is invalid." () ; error Api_errors.invalid_gpgkey_path ["gpgkey_path"] ~doc:"The GPG public key file name in the repository is invalid." () ; + error Api_errors.cdn_token_invalid ["message"] + ~doc:"The provided CDN token or token_id is empty." () ; error Api_errors.repository_already_exists ["ref"] ~doc:"The repository already exists." () ; error Api_errors.bundle_repository_already_exists ["ref"] @@ -1926,6 +1928,16 @@ let _ = "If the bundle repository or remote_pool repository is enabled, it \ should be the only one enabled repository of the pool." () ; + error Api_errors.update_syncing_remote_pool_coordinator_connection_failed [] + ~doc: + "There was an error connecting to the remote pool coordinator while \ + syncing updates from it." + () ; + error Api_errors.update_syncing_remote_pool_coordinator_service_failed [] + ~doc: + "There was an error connecting to the server while syncing updates from \ + it. The service contacted didn't reply properly." + () ; error Api_errors.repository_is_in_use [] ~doc:"The repository is in use." () ; error Api_errors.repository_cleanup_failed [] ~doc:"Failed to clean up local repository on coordinator." () ; diff --git a/ocaml/idl/datamodel_pool.ml b/ocaml/idl/datamodel_pool.ml index ab0d1669788..a86a3ea152c 100644 --- a/ocaml/idl/datamodel_pool.ml +++ b/ocaml/idl/datamodel_pool.ml @@ -1282,6 +1282,20 @@ let sync_updates = ; param_release= numbered_release "1.329.0" ; param_default= Some (VString "") } + ; { + param_type= String + ; param_name= "username" + ; param_doc= "The username of the remote pool" + ; param_release= numbered_release "24.39.0-next" + ; param_default= Some (VString "") + } + ; { + param_type= String + ; param_name= "password" + ; param_doc= "The password of the remote pool" + ; param_release= numbered_release "24.39.0-next" + ; param_default= Some (VString "") + } ] ~result:(String, "The SHA256 hash of updateinfo.xml.gz") ~allowed_roles:(_R_POOL_OP ++ _R_CLIENT_CERT) diff --git a/ocaml/xapi-cli-server/cli_frontend.ml b/ocaml/xapi-cli-server/cli_frontend.ml index c4a5a4a5dc2..4b5fa9476ae 100644 --- a/ocaml/xapi-cli-server/cli_frontend.ml +++ b/ocaml/xapi-cli-server/cli_frontend.ml @@ -511,7 +511,7 @@ let rec cmdtable_data : (string * cmd_spec) list = ; ( "pool-sync-updates" , { reqd= [] - ; optn= ["force"; "token"; "token-id"] + ; optn= ["force"; "token"; "token-id"; "username"; "password"] ; help= "Sync updates from remote YUM repository, pool-wide." ; implementation= No_fd Cli_operations.pool_sync_updates ; flags= [] diff --git a/ocaml/xapi-cli-server/cli_operations.ml b/ocaml/xapi-cli-server/cli_operations.ml index 62a655b9564..8c9a1dbaf2b 100644 --- a/ocaml/xapi-cli-server/cli_operations.ml +++ b/ocaml/xapi-cli-server/cli_operations.ml @@ -1833,8 +1833,11 @@ let pool_sync_updates printer rpc session_id params = let force = get_bool_param params "force" in let token = get_param params "token" ~default:"" in let token_id = get_param params "token-id" ~default:"" in + let username = get_param params "username" ~default:"" in + let password = get_param params "password" ~default:"" in let hash = Client.Pool.sync_updates ~rpc ~session_id ~self:pool ~force ~token ~token_id + ~username ~password in printer (Cli_printer.PList [hash]) diff --git a/ocaml/xapi-consts/api_errors.ml b/ocaml/xapi-consts/api_errors.ml index 6e9b7fdbe06..f76abc6c888 100644 --- a/ocaml/xapi-consts/api_errors.ml +++ b/ocaml/xapi-consts/api_errors.ml @@ -1314,6 +1314,8 @@ let invalid_base_url = add_error "INVALID_BASE_URL" let invalid_gpgkey_path = add_error "INVALID_GPGKEY_PATH" +let cdn_token_invalid = add_error "CDN_TOKEN_INVALID" + let repository_already_exists = add_error "REPOSITORY_ALREADY_EXISTS" let bundle_repository_already_exists = @@ -1330,6 +1332,12 @@ let can_not_periodic_sync_updates = add_error "CAN_NOT_PERIODIC_SYNC_UPDATES" let repo_should_be_single_one_enabled = add_error "REPO_SHOULD_BE_SINGLE_ONE_ENABLED" +let update_syncing_remote_pool_coordinator_connection_failed = + add_error "UPDATE_SYNCING_REMOTE_POOL_COORDINATOR_CONNECTION_FAILED" + +let update_syncing_remote_pool_coordinator_service_failed = + add_error "UPDATE_SYNCING_REMOTE_POOL_COORDINATOR_SERVICE_FAILED" + let repository_is_in_use = add_error "REPOSITORY_IS_IN_USE" let repository_cleanup_failed = add_error "REPOSITORY_CLEANUP_FAILED" diff --git a/ocaml/xapi/helpers.ml b/ocaml/xapi/helpers.ml index aff7f383e46..6bd81a84fa1 100644 --- a/ocaml/xapi/helpers.ml +++ b/ocaml/xapi/helpers.ml @@ -2030,19 +2030,24 @@ let with_temp_file ?mode prefix suffix f = let path, channel = Filename.open_temp_file ?mode prefix suffix in finally (fun () -> f (path, channel)) (fun () -> Unix.unlink path) +let with_temp_file_of_content ?mode prefix suffix content f = + let@ temp_file, temp_out_ch = with_temp_file ?mode prefix suffix in + Xapi_stdext_pervasives.Pervasiveext.finally + (fun () -> output_string temp_out_ch content) + (fun () -> close_out temp_out_ch) ; + f temp_file + let with_temp_out_ch_of_temp_file ?mode prefix suffix f = let@ path, channel = with_temp_file ?mode prefix suffix in f (path, channel |> with_temp_out_ch) -let make_external_host_verified_rpc ~__context ext_host_address ext_host_cert - xml = - let@ temp_file, temp_out_ch = with_temp_file "external-host-cert" ".pem" in - Xapi_stdext_pervasives.Pervasiveext.finally - (fun () -> output_string temp_out_ch ext_host_cert) - (fun () -> close_out temp_out_ch) ; +let make_external_host_verified_rpc ~__context host_address host_cert xml = + let@ cert_file = + with_temp_file_of_content "external-host-cert-" ".pem" host_cert + in make_remote_rpc ~__context - ~verify_cert:(Stunnel_client.external_host temp_file) - ext_host_address xml + ~verify_cert:(Stunnel_client.external_host cert_file) + host_address xml module FileSys : sig (* bash-like interface for manipulating files *) diff --git a/ocaml/xapi/pool_periodic_update_sync.ml b/ocaml/xapi/pool_periodic_update_sync.ml index a9755d0cf1e..07bb44965d7 100644 --- a/ocaml/xapi/pool_periodic_update_sync.ml +++ b/ocaml/xapi/pool_periodic_update_sync.ml @@ -140,7 +140,8 @@ let rec update_sync () = ignore (Client.Pool.sync_updates ~rpc ~session_id ~self:(Helpers.get_pool ~__context) - ~force:false ~token:"" ~token_id:"" + ~force:false ~token:"" ~token_id:"" ~username:"" + ~password:"" ) with e -> let exc = Printexc.to_string e in diff --git a/ocaml/xapi/repository.ml b/ocaml/xapi/repository.ml index 211cd4ac535..8e76ef4304f 100644 --- a/ocaml/xapi/repository.ml +++ b/ocaml/xapi/repository.ml @@ -149,26 +149,70 @@ let get_proxy_params ~__context repo_name = | _ -> ("", "", "") -let sync ~__context ~self ~token ~token_id = +type client_proxy_conf = { + cert: string + ; remote_addr: string + ; remote_port: int + ; local_host: string + ; local_port: int +} + +type client_auth_conf = + | CdnTokenAuthConf of {token: string; token_id: string} + | PoolExtHostAuthConf of { + cert: string + ; remote_addr: string + ; username: string + ; password: string + } + +let sync ~__context ~self ~token ~token_id ~username ~password = try let repo_name = get_remote_repository_name ~__context ~self in remove_repo_conf_file repo_name ; - let binary_url, source_url = - match Db.Repository.get_origin ~__context ~self with + let origin = Db.Repository.get_origin ~__context ~self in + let local_host = "127.0.0.1" in + + let ( binary_url + , source_url + , client_auth_config + , client_proxy_config ) = + match origin with | `remote -> ( Db.Repository.get_binary_url ~__context ~self , Some (Db.Repository.get_source_url ~__context ~self) + , Some (CdnTokenAuthConf {token; token_id}) + , None ) | `bundle -> let uri = Uri.make ~scheme:"file" ~path:!Xapi_globs.bundle_repository_dir () in - (Uri.to_string uri, None) + (Uri.to_string uri, None, None, None) | `remote_pool -> - (* TODO: sync with Stunnel.with_client_proxy as otherwise yum - reposync will fail when checking the self signed certificate on - the remote pool. *) - ("", None) + let uri = + Uri.make ~scheme:"http" ~host:local_host + ~port:!Xapi_globs.local_yum_repo_port + ~path:Constants.get_enabled_repository_uri () + in + let cert = Db.Repository.get_certificate ~__context ~self in + let remote_addr = + Db.Repository.get_binary_url ~__context ~self + |> Repository_helpers.get_remote_pool_coordinator_ip + in + let local_port = !Xapi_globs.local_yum_repo_port in + ( Uri.to_string uri + , None + , Some (PoolExtHostAuthConf {cert; remote_addr; username; password}) + , Some + { + cert + ; remote_addr + ; remote_port= Constants.default_ssl_port + ; local_host + ; local_port + } + ) in let gpgkey_path = match Db.Repository.get_gpgkey_path ~__context ~self with @@ -188,39 +232,105 @@ let sync ~__context ~self ~token ~token_id = Xapi_stdext_unix.Unixext.rm_rec (get_repo_config repo_name "gpgdir") ; Xapi_stdext_pervasives.Pervasiveext.finally (fun () -> - with_access_token ~token ~token_id @@ fun token_path -> - (* Configure proxy and token *) - let token_param = - match token_path with - | Some p -> - Printf.sprintf "--setopt=%s.accesstoken=file://%s" repo_name p - | None -> - "" - in - let proxy_url_param, proxy_username_param, proxy_password_param = - get_proxy_params ~__context repo_name - in - let Pkg_mgr.{cmd; params} = - [ - "--save" - ; proxy_url_param - ; proxy_username_param - ; proxy_password_param - ; token_param - ] - |> fun config -> Pkgs.config_repo ~repo_name ~config + let config_repo token_path yum_plugin = + (* Configure proxy and token *) + let token_param = + match token_path with + | "" -> + "" + | p -> + Printf.sprintf "--setopt=%s.%s=%s" repo_name yum_plugin + (Uri.make ~scheme:"file" ~path:p () |> Uri.to_string) + in + let proxy_url_param, proxy_username_param, proxy_password_param = + get_proxy_params ~__context repo_name + in + let Pkg_mgr.{cmd; params} = + [ + "--save" + ; proxy_url_param + ; proxy_username_param + ; proxy_password_param + ; token_param + ] + |> fun config -> Pkgs.config_repo ~repo_name ~config + in + ignore (Helpers.call_script ~log_output:Helpers.On_failure cmd params) in - ignore (Helpers.call_script ~log_output:Helpers.On_failure cmd params) ; - (* Import YUM repository GPG key to check metadata in reposync *) - let Pkg_mgr.{cmd; params} = Pkgs.make_cache ~repo_name in - ignore (Helpers.call_script cmd params) ; + let make_cache () = + (* Import YUM repository GPG key to check metadata in reposync *) + let Pkg_mgr.{cmd; params} = Pkgs.make_cache ~repo_name in + ignore (Helpers.call_script cmd params) + in (* Sync with remote repository *) - let Pkg_mgr.{cmd; params} = Pkgs.sync_repo ~repo_name in - Unixext.mkdir_rec !Xapi_globs.local_pool_repo_dir 0o700 ; - clean_yum_cache repo_name ; - ignore (Helpers.call_script cmd params) + let sync_repo () = + let Pkg_mgr.{cmd; params} = Pkgs.sync_repo ~repo_name in + Unixext.mkdir_rec !Xapi_globs.local_pool_repo_dir 0o700 ; + clean_yum_cache repo_name ; + ignore (Helpers.call_script cmd params) + in + + match client_auth_config with + | Some cfg -> ( + let auth, yum_plugin = + match cfg with + | CdnTokenAuthConf {token; token_id} -> + (CdnTokenAuth (token_id, token), "accesstoken") + | PoolExtHostAuthConf {cert; remote_addr; username; password} -> + let verified_rpc = + try + Helpers.make_external_host_verified_rpc ~__context + remote_addr cert + with Xmlrpc_client.Connection_reset -> + raise + (Api_errors.Server_error + ( Api_errors + .update_syncing_remote_pool_coordinator_connection_failed + , [] + ) + ) + in + let session_id = + try + Client.Client.Session.login_with_password + ~rpc:verified_rpc ~uname:username ~pwd:password + ~version:Datamodel_common.api_version_string + ~originator:Xapi_version.xapi_user_agent + with + | Http_client.Http_request_rejected _ + | Http_client.Http_error _ + -> + raise + (Api_errors.Server_error + ( Api_errors + .update_syncing_remote_pool_coordinator_service_failed + , [] + ) + ) + in + let xapi_token = session_id |> Ref.string_of in + (ExtHostAuth xapi_token, "xapitoken") + in + with_sync_client_auth auth @@ fun token_path -> + config_repo token_path yum_plugin ; + match client_proxy_config with + | None -> + make_cache () ; sync_repo () + | Some {cert; remote_addr; remote_port; local_host; local_port} -> + let ( let@ ) f x = f x in + let@ temp_file = + Helpers.with_temp_file_of_content "external-host-cert-" ".pem" + cert + in + Stunnel.with_client_proxy + ~verify_cert:(Stunnel_client.external_host temp_file) + ~remote_host:remote_addr ~remote_port ~local_host ~local_port + @@ fun () -> make_cache () ; sync_repo () + ) + | None -> + make_cache () ; sync_repo () ) (fun () -> (* Rewrite repo conf file as initial content to remove credential @@ -239,10 +349,13 @@ let sync ~__context ~self ~token ~token_id = // "repomd.xml.asc" in Sys.file_exists repo_gpg_signature - with e -> - error "Failed to sync with remote YUM repository: %s" - (ExnHelper.string_of_exn e) ; - raise Api_errors.(Server_error (reposync_failed, [])) + with + | Api_errors.Server_error (_, _) as e -> + raise e + | e -> + error "Failed to sync with remote YUM repository: %s" + (ExnHelper.string_of_exn e) ; + raise Api_errors.(Server_error (reposync_failed, [])) let http_get_host_updates_in_json ~__context ~host ~installed = let host_session_id = diff --git a/ocaml/xapi/repository.mli b/ocaml/xapi/repository.mli index 5e1c78690fb..3049c003400 100644 --- a/ocaml/xapi/repository.mli +++ b/ocaml/xapi/repository.mli @@ -48,6 +48,8 @@ val sync : -> self:[`Repository] API.Ref.t -> token:string -> token_id:string + -> username:string + -> password:string -> bool val create_pool_repository : diff --git a/ocaml/xapi/repository_helpers.ml b/ocaml/xapi/repository_helpers.ml index 4016a158237..dc08f35ac3d 100644 --- a/ocaml/xapi/repository_helpers.ml +++ b/ocaml/xapi/repository_helpers.ml @@ -231,18 +231,22 @@ let assert_gpgkey_path_is_valid path = raise Api_errors.(Server_error (invalid_gpgkey_path, [path])) ) -let assert_remote_pool_url_is_valid ~url = +let get_remote_pool_coordinator_ip url = let uri = Uri.of_string url in match (Uri.scheme uri, Uri.host uri, Uri.path uri) with | Some "https", Some host, path when path = Constants.get_enabled_repository_uri && Helpers.is_valid_ip `ipv4or6 host -> - () + host | _ -> error "Invalid url: %s, expected url format: %s" url ("https://" ^ Constants.get_enabled_repository_uri) ; raise Api_errors.(Server_error (invalid_base_url, [url])) +let assert_remote_pool_url_is_valid ~url = + get_remote_pool_coordinator_ip url + |> Xapi_stdext_pervasives.Pervasiveext.ignore_string + let with_pool_repositories f = Xapi_stdext_pervasives.Pervasiveext.finally (fun () -> @@ -1284,26 +1288,32 @@ let get_single_enabled_update_repository ~__context = in get_singleton enabled_update_repositories -let with_access_token ~token ~token_id f = - match (token, token_id) with - | t, tid when t <> "" && tid <> "" -> - info "sync updates with token_id: %s" tid ; - let json = `Assoc [("token", `String t); ("token_id", `String tid)] in - let tmpfile, tmpch = - Filename.open_temp_file ~mode:[Open_text] "accesstoken" ".json" - in - Xapi_stdext_pervasives.Pervasiveext.finally - (fun () -> - output_string tmpch (Yojson.Basic.to_string json) ; - close_out tmpch ; - f (Some tmpfile) - ) - (fun () -> Unixext.unlink_safe tmpfile) - | t, tid when t = "" && tid = "" -> - f None - | _ -> - let msg = Printf.sprintf "%s: The token or token_id is empty" __LOC__ in - raise Api_errors.(Server_error (internal_error, [msg])) +type client_auth = ExtHostAuth of string | CdnTokenAuth of string * string + +let with_sync_client_auth auth f = + let secret = + match auth with + | ExtHostAuth session -> + Some (`Assoc [("xapitoken", `String session)]) + | CdnTokenAuth (token_id, token) when token_id = "" && token = "" -> + None + | CdnTokenAuth (token_id, token) when token_id <> "" && token <> "" -> + Some (`Assoc [("token", `String token); ("token_id", `String token_id)]) + | CdnTokenAuth (token_id, _) -> + let msg = + if token_id = "" then + Printf.sprintf "%s: The token_id is empty" __LOC__ + else + Printf.sprintf "%s: The token is empty" __LOC__ + in + raise Api_errors.(Server_error (cdn_token_invalid, [msg])) + in + match secret with + | Some s -> + Helpers.with_temp_file_of_content ~mode:[Open_text] "token" ".json" + (Yojson.Basic.to_string s) f + | None -> + f "" let prune_updateinfo_for_livepatches latest_lps updateinfo = let livepatches = diff --git a/ocaml/xapi/xapi_pool.ml b/ocaml/xapi/xapi_pool.ml index 5eec626c601..9b7953306dc 100644 --- a/ocaml/xapi/xapi_pool.ml +++ b/ocaml/xapi/xapi_pool.ml @@ -3559,12 +3559,15 @@ let remove_repository ~__context ~self ~value = if Db.Pool.get_repositories ~__context ~self = [] then Db.Pool.set_last_update_sync ~__context ~self ~value:Date.epoch -let sync_repos ~__context ~self ~repos ~force ~token ~token_id = +let sync_repos ~__context ~self ~repos ~force ~token ~token_id ~username + ~password = let open Repository in repos |> List.iter (fun repo -> if force then cleanup_pool_repo ~__context ~self:repo ; - let complete = sync ~__context ~self:repo ~token ~token_id in + let complete = + sync ~__context ~self:repo ~token ~token_id ~username ~password + in (* Dnf and custom yum-utils sync all the metadata including updateinfo, * Thus no need to re-create pool repository *) if Pkgs.manager = Yum && complete = false then @@ -3574,14 +3577,14 @@ let sync_repos ~__context ~self ~repos ~force ~token ~token_id = Db.Pool.set_last_update_sync ~__context ~self ~value:(Date.now ()) ; checksum -let sync_updates ~__context ~self ~force ~token ~token_id = +let sync_updates ~__context ~self ~force ~token ~token_id ~username ~password = Pool_features.assert_enabled ~__context ~f:Features.Updates ; Xapi_pool_helpers.with_pool_operation ~__context ~self ~doc:"pool.sync_updates" ~op:`sync_updates @@ fun () -> let repos = Repository_helpers.get_enabled_repositories ~__context in assert_can_sync_updates ~__context ~repos ; - sync_repos ~__context ~self ~repos ~force ~token ~token_id + sync_repos ~__context ~self ~repos ~force ~token ~token_id ~username ~password let check_update_readiness ~__context ~self:_ ~requires_reboot = (* Pool license check *) @@ -3956,7 +3959,7 @@ let put_bundle_handler (req : Request.t) s _ = (fun () -> try sync_repos ~__context ~self:pool ~repos:[repo] ~force:true - ~token:"" ~token_id:"" + ~token:"" ~token_id:"" ~username:"" ~password:"" |> ignore with _ -> raise Api_errors.(Server_error (bundle_sync_failed, [])) diff --git a/ocaml/xapi/xapi_pool.mli b/ocaml/xapi/xapi_pool.mli index 835a356f782..494a486032b 100644 --- a/ocaml/xapi/xapi_pool.mli +++ b/ocaml/xapi/xapi_pool.mli @@ -360,6 +360,8 @@ val sync_updates : -> force:bool -> token:string -> token_id:string + -> username:string + -> password:string -> string val check_update_readiness : diff --git a/python3/dnf_plugins/accesstoken.py b/python3/dnf_plugins/accesstoken.py index 2537d2a6721..97635fa160b 100644 --- a/python3/dnf_plugins/accesstoken.py +++ b/python3/dnf_plugins/accesstoken.py @@ -10,7 +10,7 @@ class InvalidToken(Exception): - """Token is invlaid""" + """Token is invalid""" def __init__(self, token): super().__init__(f"Invalid token: {token}") diff --git a/python3/dnf_plugins/ptoken.py b/python3/dnf_plugins/ptoken.py index c2ea73fccc8..35b6f9aef70 100644 --- a/python3/dnf_plugins/ptoken.py +++ b/python3/dnf_plugins/ptoken.py @@ -24,6 +24,8 @@ def config(self): for repo_name in self.base.repos: repo = self.base.repos[repo_name] + # Only include the ptoken for repos with a localhost URL, for added safety. + # These will be proxied to the coordinator through stunnel, set up by xapi. if len(repo.baseurl) > 0 and repo.baseurl[0].startswith("http://127.0.0.1") \ and repo.ptoken: secret = "pool_secret=" + ptoken diff --git a/python3/dnf_plugins/xapitoken.py b/python3/dnf_plugins/xapitoken.py new file mode 100644 index 00000000000..377fe33964e --- /dev/null +++ b/python3/dnf_plugins/xapitoken.py @@ -0,0 +1,49 @@ +"""dnf plugin to set xapitoken http header for enabled repos""" +import json +import logging +# Disable the error, it can be import in production env +# and mocked out in unitttest +# pylint: disable=import-error +# pytype: disable=import-error +import dnf +import urlgrabber + + +class InvalidToken(Exception): + """Token is invalid""" + def __init__(self, token): + super().__init__(f"Invalid token: {token}") + + +#pylint: disable=too-few-public-methods +class XapiToken(dnf.Plugin): + """dnf xapitoken plugin class""" + + name = "xapitoken" + + def config(self): + """ DNF plugin config hook, + refer to https://dnf.readthedocs.io/en/latest/api_plugins.html""" + + for repo_name in self.base.repos: + repo = self.base.repos[repo_name] + + token_url = repo.xapitoken + if not token_url or token_url == '': + continue + try: + token_str = urlgrabber.urlopen(token_url).read().strip() + token = json.loads(token_str) + except Exception: #pylint: disable=broad-except + logging.debug("Failed to load token from: %s", token_url) + continue + + if not token.get('xapitoken'): + raise InvalidToken(token) + + # Only include the xapitoken for repos with a localhost URL, for added safety. + # These will be proxied to the remote pool coordinator through stunnel, set up by xapi. + if len(repo.baseurl) > 0 and repo.baseurl[0].startswith("http://127.0.0.1") \ + and repo.xapitoken: + secret = "session_id=" + str(token["xapitoken"]) + repo.set_http_headers([f'cookie:{secret}']) diff --git a/python3/tests/test_dnf_plugins.py b/python3/tests/test_dnf_plugins.py index 2f82b1eb5cb..895317f8778 100644 --- a/python3/tests/test_dnf_plugins.py +++ b/python3/tests/test_dnf_plugins.py @@ -1,4 +1,4 @@ -"""Test module for dnf accesstoken""" +"""Test module for dnf accesstoken, ptoken and xapitoken""" import unittest import sys import json @@ -17,14 +17,16 @@ accesstoken = import_file_as_module("python3/dnf_plugins/accesstoken.py") ptoken = import_file_as_module("python3/dnf_plugins/ptoken.py") +xapitoken = import_file_as_module("python3/dnf_plugins/xapitoken.py") REPO_NAME = "testrepo" -def _mock_repo(a_token=None, p_token=None, baseurl=None): +def _mock_repo(a_token=None, p_token=None, xapi_token=None, baseurl=None): mock_repo = MagicMock() mock_repo.accesstoken = a_token mock_repo.ptoken = p_token + mock_repo.xapitoken = xapi_token mock_repo.baseurl = baseurl mock_base = MagicMock() mock_base.repos = {REPO_NAME: mock_repo} @@ -103,3 +105,60 @@ def test_local_repo_does_not_enable_ptoken_should_ignore_ptoken(self, mock_open) mock_repo = _mock_repo(p_token=False, baseurl=["http://127.0.0.1/some_local_path"]) ptoken.Ptoken(mock_repo.base, MagicMock()).config() assert not mock_repo.set_http_headers.called + +@patch("xapitoken.urlgrabber") +class TestXapitoken(unittest.TestCase): + """Test class for xapitoken dnf plugin""" + + def test_set_http_header_with_xapi_token(self, mock_grabber): + """test config succeed with xapitokan""" + mock_repo = _mock_repo(xapi_token="file:///mock_xapitoken_url", + baseurl=["http://127.0.0.1/some_local_path"]) + mock_grabber.urlopen.return_value.read.return_value = json.dumps({ + "xapitoken": "valid_token", + }) + xapitoken.XapiToken(mock_repo.base, MagicMock()).config() + mock_repo.set_http_headers.assert_called_with( + ['cookie:session_id=valid_token'] + ) + + def test_repo_without_xapi_token(self, mock_grabber): + """If repo has not xapitoken, it should not be blocked""" + mock_repo = _mock_repo() + xapitoken.XapiToken(mock_repo.base, MagicMock()).config() + assert not mock_repo.set_http_headers.called + + def test_ignore_invalid_token_url(self, mock_grabber): + """If repo provided an invalid token url, it should be ignored""" + mock_repo = _mock_repo(xapi_token="Not_existed") + xapitoken.XapiToken(mock_repo.base, MagicMock()).config() + assert not mock_repo.set_http_headers.called + + def test_invalid_token_raise_exception(self, mock_grabber): + """Token with right json format, bad content should raise""" + mock_repo = _mock_repo(xapi_token="file:///file_contain_invalid_token", + baseurl=["http://127.0.0.1/some_local_path"]) + mock_grabber.urlopen.return_value.read.return_value = json.dumps({ + "bad_token": "I am bad guy" + }) + with self.assertRaises(xapitoken.InvalidToken): + xapitoken.XapiToken(mock_repo.base, MagicMock()).config() + + def test_remote_repo_ignore_xapitoken(self, mock_grabber): + """non-local repo should just ignore the xapitoken""" + mock_repo = _mock_repo(xapi_token=True, + baseurl=["http://some_remote_token/some_local_path"]) + mock_grabber.urlopen.return_value.read.return_value = json.dumps({ + "xapitoken": "valid_token", + }) + xapitoken.XapiToken(mock_repo.base, MagicMock()).config() + assert not mock_repo.set_http_headers.called + + def test_local_repo_does_not_enable_xapitoken_should_ignore_xapitoken(self, mock_grabber): + """local repo which has not enabled xapitoken should just ignore the xapitoken""" + mock_repo = _mock_repo(xapi_token=False, baseurl=["http://127.0.0.1/some_local_path"]) + mock_grabber.urlopen.return_value.read.return_value = json.dumps({ + "xapitoken": "valid_token", + }) + xapitoken.XapiToken(mock_repo.base, MagicMock()).config() + assert not mock_repo.set_http_headers.called diff --git a/scripts/Makefile b/scripts/Makefile index 503e7838546..6bb740e9df8 100644 --- a/scripts/Makefile +++ b/scripts/Makefile @@ -143,6 +143,8 @@ install: # YUM plugins $(IPROG) yum-plugins/accesstoken.py $(DESTDIR)$(YUMPLUGINDIR) $(IDATA) yum-plugins/accesstoken.conf $(DESTDIR)$(YUMPLUGINCONFDIR) + $(IPROG) yum-plugins/xapitoken.py $(DESTDIR)$(YUMPLUGINDIR) + $(IDATA) yum-plugins/xapitoken.conf $(DESTDIR)$(YUMPLUGINCONFDIR) $(IPROG) yum-plugins/ptoken.py $(DESTDIR)$(YUMPLUGINDIR) $(IDATA) yum-plugins/ptoken.conf $(DESTDIR)$(YUMPLUGINCONFDIR) # maillanguages diff --git a/scripts/yum-plugins/accesstoken.py b/scripts/yum-plugins/accesstoken.py index 0f549f27121..a83d7bf2ce9 100644 --- a/scripts/yum-plugins/accesstoken.py +++ b/scripts/yum-plugins/accesstoken.py @@ -11,16 +11,16 @@ # The content of the file referred by the looks like: # { 'token': '...', 'token_id': '...' } +import json from yum import config from yum.plugins import TYPE_CORE -import json import urlgrabber requires_api_version = '2.5' plugin_type = (TYPE_CORE,) -def config_hook(conduit): +def config_hook(conduit): # pylint: disable=unused-argument config.RepoConf.accesstoken = config.UrlOption() def init_hook(conduit): @@ -35,11 +35,11 @@ def init_hook(conduit): try: token_str = urlgrabber.urlopen(token_url).read().strip() token = json.loads(token_str) - except: + except Exception: #pylint: disable=broad-except continue if not (token['token'] and token['token_id']): - raise Exception("Invalid token or token_id") + raise Exception("Invalid token or token_id") #pylint: disable=broad-exception-raised repo.http_headers['X-Access-Token'] = str(token['token']) repo.http_headers['Referer'] = str(token['token_id']) diff --git a/scripts/yum-plugins/ptoken.py b/scripts/yum-plugins/ptoken.py index 74536e19ee8..0bc0cca0a6d 100755 --- a/scripts/yum-plugins/ptoken.py +++ b/scripts/yum-plugins/ptoken.py @@ -25,7 +25,7 @@ def init_hook(conduit): for name in repos.repos: repo = repos.repos[name] # Only include the ptoken for repos with a localhost URL, for added safety. - # These may be proxied to the coordinator through stunnel, set up by xapi. + # These will be proxied to the coordinator through stunnel, set up by xapi. if len(repo.baseurl) > 0 and repo.baseurl[0].startswith("http://127.0.0.1") \ and repo.getConfigOption('ptoken'): repo.http_headers['cookie'] = "pool_secret=" + ptoken diff --git a/scripts/yum-plugins/xapitoken.conf b/scripts/yum-plugins/xapitoken.conf new file mode 100644 index 00000000000..8e4d76c728b --- /dev/null +++ b/scripts/yum-plugins/xapitoken.conf @@ -0,0 +1,2 @@ +[main] +enabled=1 diff --git a/scripts/yum-plugins/xapitoken.py b/scripts/yum-plugins/xapitoken.py new file mode 100644 index 00000000000..6b959c74462 --- /dev/null +++ b/scripts/yum-plugins/xapitoken.py @@ -0,0 +1,48 @@ +#!/usr/bin/python + +# Drop this file into /usr/lib/yum-plugins/ +# Enable it by creating conf file /etc/yum/pluginconf.d/xapitoken.conf: +# [main] +# enabled=1 +# +# Configure it by: +# yum-config-manager --setopt=.xapitoken=file:// --save + +# The content of the file referred by the looks like: +# { 'xapitoken': '...' } + +import json +from yum import config +from yum.plugins import TYPE_CORE +import urlgrabber + + +requires_api_version = '2.5' +plugin_type = (TYPE_CORE,) + +def config_hook(conduit): # pylint: disable=unused-argument + config.RepoConf.xapitoken = config.UrlOption() + +def init_hook(conduit): + repos = conduit.getRepos() + for name in repos.repos: + repo = repos.repos[name] + token_url = repo.getConfigOption('xapitoken') + if not token_url or token_url == '': + continue + + token = {} + try: + token_str = urlgrabber.urlopen(token_url).read().strip() + token = json.loads(token_str) + except Exception: #pylint: disable=broad-except + continue + + if not token['xapitoken']: + raise Exception("Invalid xapitoken") #pylint: disable=broad-exception-raised + + # Only include the xapitoken for repos with a localhost URL, for added safety. + # These will be proxied to the remote pool coordinator through stunnel, set up by xapi. + if len(repo.baseurl) > 0 and repo.baseurl[0].startswith("http://127.0.0.1") \ + and repo.getConfigOption('xapitoken'): + repo.http_headers['cookie'] = "session_id=" + str(token['xapitoken'])