[2024-11-22T00:40:10.641Z] 19:40:10 - MainThread - ocs_ci.framework.pytest_customization.reports - �[32mINFO�[0m  - duration reported by tests/functional/object/mcg/test_nsfs.py::TestNSFSObjectIntegrity::test_nsfs_object_integrity[OC-20Gi-Export] immediately after test execution: 12.48
[2024-11-22T00:40:10.641Z] �[31mFAILED�[0m
[2024-11-22T00:40:10.641Z] ______ TestNSFSObjectIntegrity.test_nsfs_object_integrity[OC-20Gi-Export] ______
[2024-11-22T00:40:10.641Z] 
[2024-11-22T00:40:10.641Z] self = <mcg.test_nsfs.TestNSFSObjectIntegrity object at 0x7f6258af17f0>
[2024-11-22T00:40:10.641Z] nsfs_bucket_factory = <function nsfs_bucket_factory_fixture.<locals>.nsfs_bucket_factory_implementation at 0x7f62586114c0>
[2024-11-22T00:40:10.641Z] awscli_pod_session = <ocs_ci.ocs.resources.pod.Pod object at 0x7f6259a8f400>
[2024-11-22T00:40:10.641Z] test_directory_setup = SetupDirs(origin_dir='test_nsfs_object_integrity[OC-20Gi-Export]/origin', result_dir='test_nsfs_object_integrity[OC-20Gi-Export]/result')
[2024-11-22T00:40:10.641Z] nsfs_obj = NSFS(method='OC', pvc_name=None, pvc_size=20, sub_path=None, fs_backend=None, mount_existing_dir=True, existing_dir_mo...ted_bucket_path=None, s3_creds=None, nss=<ocs_ci.ocs.resources.namespacestore.NamespaceStore object at 0x7f6259a968e0>)
[2024-11-22T00:40:10.641Z] 
[2024-11-22T00:40:10.641Z]     @pytest.mark.polarion_id("OCS-3735")
[2024-11-22T00:40:10.641Z]     @pytest.mark.parametrize(
[2024-11-22T00:40:10.641Z]         argnames="nsfs_obj",
[2024-11-22T00:40:10.641Z]         argvalues=[
[2024-11-22T00:40:10.641Z]             pytest.param(
[2024-11-22T00:40:10.641Z]                 NSFS(
[2024-11-22T00:40:10.641Z]                     method="CLI",
[2024-11-22T00:40:10.641Z]                     pvc_size=25,
[2024-11-22T00:40:10.641Z]                 ),
[2024-11-22T00:40:10.641Z]                 marks=[tier1],
[2024-11-22T00:40:10.641Z]             ),
[2024-11-22T00:40:10.641Z]             pytest.param(
[2024-11-22T00:40:10.641Z]                 NSFS(
[2024-11-22T00:40:10.641Z]                     method="OC",
[2024-11-22T00:40:10.641Z]                     pvc_size=20,
[2024-11-22T00:40:10.641Z]                     mount_existing_dir=True,
[2024-11-22T00:40:10.641Z]                 ),
[2024-11-22T00:40:10.641Z]                 marks=[tier1],
[2024-11-22T00:40:10.641Z]             ),
[2024-11-22T00:40:10.641Z]         ],
[2024-11-22T00:40:10.641Z]         ids=[
[2024-11-22T00:40:10.641Z]             "CLI-25Gi",
[2024-11-22T00:40:10.641Z]             "OC-20Gi-Export",
[2024-11-22T00:40:10.641Z]         ],
[2024-11-22T00:40:10.641Z]     )
[2024-11-22T00:40:10.641Z]     def test_nsfs_object_integrity(
[2024-11-22T00:40:10.641Z]         self, nsfs_bucket_factory, awscli_pod_session, test_directory_setup, nsfs_obj
[2024-11-22T00:40:10.641Z]     ):
[2024-11-22T00:40:10.641Z]         """
[2024-11-22T00:40:10.641Z]         Test NSFS object integrity -
[2024-11-22T00:40:10.641Z]         1. Write to the NSFS bucket
[2024-11-22T00:40:10.641Z]         2. Read the objects back
[2024-11-22T00:40:10.641Z]         3. Compare their checksums
[2024-11-22T00:40:10.641Z]         4. Also compare the checksums of the files that reside on the filesystem
[2024-11-22T00:40:10.641Z]     
[2024-11-22T00:40:10.641Z]         """
[2024-11-22T00:40:10.641Z] >       nsfs_bucket_factory(nsfs_obj)
[2024-11-22T00:40:10.641Z] 
[2024-11-22T00:40:10.641Z] �[1m�[31mtests/functional/object/mcg/test_nsfs.py�[0m:71: 
[2024-11-22T00:40:10.641Z] _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
[2024-11-22T00:40:10.641Z] �[1m�[31mtests/conftest.py�[0m:5901: in nsfs_bucket_factory_implementation
[2024-11-22T00:40:10.641Z]     nsfs_deploy = nsfs_interface(nsfs_obj.nss.uls_name, nsfs_obj.mount_path)
[2024-11-22T00:40:10.641Z] �[1m�[31mtests/conftest.py�[0m:5729: in nsfs_interface_deployment_factory
[2024-11-22T00:40:10.641Z]     deployment_obj = helpers.create_resource(**nsfs_deployment_data)
[2024-11-22T00:40:10.641Z] �[1m�[31mocs_ci/helpers/helpers.py�[0m:105: in create_resource
[2024-11-22T00:40:10.641Z]     created_resource = ocs_obj.create(do_reload=do_reload)
[2024-11-22T00:40:10.641Z] �[1m�[31mocs_ci/ocs/resources/ocs.py�[0m:121: in create
[2024-11-22T00:40:10.641Z]     status = self.ocp.create(yaml_file=self.temp_yaml)
[2024-11-22T00:40:10.641Z] �[1m�[31mocs_ci/ocs/ocp.py�[0m:442: in create
[2024-11-22T00:40:10.641Z]     output = self.exec_oc_cmd(command)
[2024-11-22T00:40:10.641Z] �[1m�[31mocs_ci/ocs/ocp.py�[0m:212: in exec_oc_cmd
[2024-11-22T00:40:10.641Z]     out = run_cmd(
[2024-11-22T00:40:10.641Z] �[1m�[31mocs_ci/utility/utils.py�[0m:488: in run_cmd
[2024-11-22T00:40:10.641Z]     completed_process = exec_cmd(
[2024-11-22T00:40:10.641Z] _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
[2024-11-22T00:40:10.641Z] 
[2024-11-22T00:40:10.641Z] cmd = ['oc', '--kubeconfig', '/home/jenkins/current-cluster-dir/openshift-cluster-dir/auth/kubeconfig', '-n', 'openshift-storage', 'create', ...]
[2024-11-22T00:40:10.641Z] secrets = None, timeout = 600, ignore_error = False, threading_lock = None
[2024-11-22T00:40:10.641Z] silent = False, use_shell = False
[2024-11-22T00:40:10.641Z] cluster_config = <ocs_ci.framework.MultiClusterConfig object at 0x7f628b297490>
[2024-11-22T00:40:10.641Z] lock_timeout = 7200, kwargs = {}
[2024-11-22T00:40:10.641Z] masked_cmd = 'oc -n openshift-storage create -f /tmp/Deploymentoifophxs -o yaml'
[2024-11-22T00:40:10.641Z] kubepath = '/home/jenkins/current-cluster-dir/openshift-cluster-dir/auth/kubeconfig'
[2024-11-22T00:40:10.641Z] kube_index = 1, plugin_list = 'oc plugin list'
[2024-11-22T00:40:10.641Z] cp = CompletedProcess(args=['oc', 'plugin', 'list'], returncode=1, stdout=b'', stderr=b'Unable to read directory "/home/jen.../jenkins/.local/bin: no such file or directory. Skipping...\nerror: unable to find any kubectl plugins in your PATH\n')
[2024-11-22T00:40:10.641Z] subcmd = '_n'
[2024-11-22T00:40:10.641Z] 
[2024-11-22T00:40:10.641Z]     def exec_cmd(
[2024-11-22T00:40:10.641Z]         cmd,
[2024-11-22T00:40:10.641Z]         secrets=None,
[2024-11-22T00:40:10.641Z]         timeout=600,
[2024-11-22T00:40:10.641Z]         ignore_error=False,
[2024-11-22T00:40:10.641Z]         threading_lock=None,
[2024-11-22T00:40:10.641Z]         silent=False,
[2024-11-22T00:40:10.641Z]         use_shell=False,
[2024-11-22T00:40:10.641Z]         cluster_config=None,
[2024-11-22T00:40:10.641Z]         lock_timeout=7200,
[2024-11-22T00:40:10.641Z]         **kwargs,
[2024-11-22T00:40:10.641Z]     ):
[2024-11-22T00:40:10.641Z]         """
[2024-11-22T00:40:10.641Z]         Run an arbitrary command locally
[2024-11-22T00:40:10.641Z]     
[2024-11-22T00:40:10.641Z]         If the command is grep and matching pattern is not found, then this function
[2024-11-22T00:40:10.641Z]         returns "command terminated with exit code 1" in stderr.
[2024-11-22T00:40:10.641Z]     
[2024-11-22T00:40:10.641Z]         Args:
[2024-11-22T00:40:10.641Z]             cmd (str): command to run
[2024-11-22T00:40:10.641Z]             secrets (list): A list of secrets to be masked with asterisks
[2024-11-22T00:40:10.641Z]                 This kwarg is popped in order to not interfere with
[2024-11-22T00:40:10.641Z]                 subprocess.run(``**kwargs``)
[2024-11-22T00:40:10.641Z]             timeout (int): Timeout for the command, defaults to 600 seconds.
[2024-11-22T00:40:10.641Z]             ignore_error (bool): True if ignore non zero return code and do not
[2024-11-22T00:40:10.641Z]                 raise the exception.
[2024-11-22T00:40:10.641Z]             threading_lock (threading.RLock): threading.RLock object that is used
[2024-11-22T00:40:10.641Z]                 for handling concurrent oc commands
[2024-11-22T00:40:10.641Z]             silent (bool): If True will silent errors from the server, default false
[2024-11-22T00:40:10.641Z]             use_shell (bool): If True will pass the cmd without splitting
[2024-11-22T00:40:10.641Z]             cluster_config (MultiClusterConfig): In case of multicluster environment this object
[2024-11-22T00:40:10.641Z]                     will be non-null
[2024-11-22T00:40:10.641Z]             lock_timeout (int): maximum timeout to wait for lock to prevent deadlocks (default 2 hours)
[2024-11-22T00:40:10.641Z]     
[2024-11-22T00:40:10.641Z]         Raises:
[2024-11-22T00:40:10.641Z]             CommandFailed: In case the command execution fails
[2024-11-22T00:40:10.641Z]     
[2024-11-22T00:40:10.641Z]         Returns:
[2024-11-22T00:40:10.641Z]             (CompletedProcess) A CompletedProcess object of the command that was executed
[2024-11-22T00:40:10.641Z]             CompletedProcess attributes:
[2024-11-22T00:40:10.641Z]             args: The list or str args passed to run().
[2024-11-22T00:40:10.641Z]             returncode (str): The exit code of the process, negative for signals.
[2024-11-22T00:40:10.641Z]             stdout     (str): The standard output (None if not captured).
[2024-11-22T00:40:10.641Z]             stderr     (str): The standard error (None if not captured).
[2024-11-22T00:40:10.641Z]     
[2024-11-22T00:40:10.641Z]         """
[2024-11-22T00:40:10.641Z]         masked_cmd = mask_secrets(cmd, secrets)
[2024-11-22T00:40:10.641Z]         log.info(f"Executing command: {masked_cmd}")
[2024-11-22T00:40:10.641Z]         if isinstance(cmd, str) and not kwargs.get("shell"):
[2024-11-22T00:40:10.641Z]             cmd = shlex.split(cmd)
[2024-11-22T00:40:10.641Z]         if config.RUN.get("custom_kubeconfig_location") and cmd[0] == "oc":
[2024-11-22T00:40:10.641Z]             if "--kubeconfig" in cmd:
[2024-11-22T00:40:10.641Z]                 cmd.pop(2)
[2024-11-22T00:40:10.641Z]                 cmd.pop(1)
[2024-11-22T00:40:10.641Z]             cmd = list_insert_at_position(cmd, 1, ["--kubeconfig"])
[2024-11-22T00:40:10.641Z]             cmd = list_insert_at_position(
[2024-11-22T00:40:10.641Z]                 cmd, 2, [config.RUN["custom_kubeconfig_location"]]
[2024-11-22T00:40:10.641Z]             )
[2024-11-22T00:40:10.641Z]         if cluster_config and cmd[0] == "oc" and "--kubeconfig" not in cmd:
[2024-11-22T00:40:10.641Z]             kubepath = cluster_config.RUN["kubeconfig"]
[2024-11-22T00:40:10.641Z]             kube_index = 1
[2024-11-22T00:40:10.641Z]             # check if we have an oc plugin in the command
[2024-11-22T00:40:10.641Z]             plugin_list = "oc plugin list"
[2024-11-22T00:40:10.641Z]             cp = subprocess.run(
[2024-11-22T00:40:10.641Z]                 shlex.split(plugin_list),
[2024-11-22T00:40:10.641Z]                 stdout=subprocess.PIPE,
[2024-11-22T00:40:10.641Z]                 stderr=subprocess.PIPE,
[2024-11-22T00:40:10.641Z]             )
[2024-11-22T00:40:10.641Z]             subcmd = cmd[1].split("-")
[2024-11-22T00:40:10.641Z]             if len(subcmd) > 1:
[2024-11-22T00:40:10.641Z]                 subcmd = "_".join(subcmd)
[2024-11-22T00:40:10.641Z]             if not isinstance(subcmd, str) and isinstance(subcmd, list):
[2024-11-22T00:40:10.641Z]                 subcmd = str(subcmd[0])
[2024-11-22T00:40:10.641Z]     
[2024-11-22T00:40:10.641Z]             for l in cp.stdout.decode().splitlines():
[2024-11-22T00:40:10.641Z]                 if subcmd in l:
[2024-11-22T00:40:10.641Z]                     # If oc cmdline has plugin name then we need to push the
[2024-11-22T00:40:10.641Z]                     # --kubeconfig to next index
[2024-11-22T00:40:10.641Z]                     kube_index = 2
[2024-11-22T00:40:10.641Z]                     log.info(f"Found oc plugin {subcmd}")
[2024-11-22T00:40:10.641Z]             cmd = list_insert_at_position(cmd, kube_index, ["--kubeconfig"])
[2024-11-22T00:40:10.641Z]             cmd = list_insert_at_position(cmd, kube_index + 1, [kubepath])
[2024-11-22T00:40:10.641Z]         try:
[2024-11-22T00:40:10.641Z]             if threading_lock and cmd[0] == "oc":
[2024-11-22T00:40:10.641Z]                 threading_lock.acquire(timeout=lock_timeout)
[2024-11-22T00:40:10.641Z]             completed_process = subprocess.run(
[2024-11-22T00:40:10.641Z]                 cmd,
[2024-11-22T00:40:10.641Z]                 stdout=subprocess.PIPE,
[2024-11-22T00:40:10.641Z]                 stderr=subprocess.PIPE,
[2024-11-22T00:40:10.641Z]                 stdin=subprocess.PIPE,
[2024-11-22T00:40:10.641Z]                 timeout=timeout,
[2024-11-22T00:40:10.641Z]                 **kwargs,
[2024-11-22T00:40:10.641Z]             )
[2024-11-22T00:40:10.641Z]         finally:
[2024-11-22T00:40:10.641Z]             if threading_lock and cmd[0] == "oc":
[2024-11-22T00:40:10.641Z]                 threading_lock.release()
[2024-11-22T00:40:10.641Z]         masked_stdout = mask_secrets(completed_process.stdout.decode(), secrets)
[2024-11-22T00:40:10.641Z]         if len(completed_process.stdout) > 0:
[2024-11-22T00:40:10.641Z]             log.debug(f"Command stdout: {masked_stdout}")
[2024-11-22T00:40:10.641Z]         else:
[2024-11-22T00:40:10.641Z]             log.debug("Command stdout is empty")
[2024-11-22T00:40:10.641Z]     
[2024-11-22T00:40:10.641Z]         masked_stderr = mask_secrets(completed_process.stderr.decode(), secrets)
[2024-11-22T00:40:10.641Z]         if len(completed_process.stderr) > 0:
[2024-11-22T00:40:10.641Z]             if not silent:
[2024-11-22T00:40:10.641Z]                 log.warning(f"Command stderr: {masked_stderr}")
[2024-11-22T00:40:10.641Z]         else:
[2024-11-22T00:40:10.641Z]             log.debug("Command stderr is empty")
[2024-11-22T00:40:10.641Z]         log.debug(f"Command return code: {completed_process.returncode}")
[2024-11-22T00:40:10.641Z]         if completed_process.returncode and not ignore_error:
[2024-11-22T00:40:10.641Z]             masked_stderr = bin_xml_escape(filter_out_emojis(masked_stderr))
[2024-11-22T00:40:10.641Z]             if (
[2024-11-22T00:40:10.641Z]                 "grep" in masked_cmd
[2024-11-22T00:40:10.641Z]                 and b"command terminated with exit code 1" in completed_process.stderr
[2024-11-22T00:40:10.641Z]             ):
[2024-11-22T00:40:10.641Z]                 log.info(f"No results found for grep command: {masked_cmd}")
[2024-11-22T00:40:10.641Z]             else:
[2024-11-22T00:40:10.641Z] >               raise CommandFailed(
[2024-11-22T00:40:10.641Z]                     f"Error during execution of command: {masked_cmd}."
[2024-11-22T00:40:10.641Z]                     f"\nError is {masked_stderr}"
[2024-11-22T00:40:10.641Z]                 )
[2024-11-22T00:40:10.641Z] �[1m�[31mE               ocs_ci.ocs.exceptions.CommandFailed: Error during execution of command: oc -n openshift-storage create -f /tmp/Deploymentoifophxs -o yaml.�[0m
[2024-11-22T00:40:10.641Z] �[1m�[31mE               Error is Error from server (NotFound): error when creating "/tmp/Deploymentoifophxs": namespaces "openshift-storage" not found�[0m
[2024-11-22T00:40:10.641Z] 
[2024-11-22T00:40:10.641Z] �[1m�[31mocs_ci/utility/utils.py�[0m:710: CommandFailed