Invalidate the repo cache when hitting 404s

[Salamandar]

tl;dr :

# incus image list yunohost:yunohost/trixie-unstable/appci
+-----------------------------------------+--------------+--------+-----------------------------------------------+--------------+-----------+-----------+----------------------+
|                  ALIAS                  | FINGERPRINT  | PUBLIC |                  DESCRIPTION                  | ARCHITECTURE |   TYPE    |   SIZE    |     UPLOAD DATE      |
+-----------------------------------------+--------------+--------+-----------------------------------------------+--------------+-----------+-----------+----------------------+
| yunohost/trixie-unstable/appci (1 more) | 500dedd061a0 | yes    | yunohost trixie-unstable amd64 (202411120627) | x86_64       | CONTAINER | 369.71MiB | 2024/11/12 01:00 CET |
+-----------------------------------------+--------------+--------+-----------------------------------------------+--------------+-----------+-----------+----------------------+

but :

# incus image copy yunohost:yunohost/trixie-unstable/appci local: --copy-aliases --auto-update --debug
DEBUG  [2024-11-14T00:12:23+01:00] Connecting to a remote simplestreams server   URL="https://repo.yunohost.org/incus/"
DEBUG  [2024-11-14T00:12:23+01:00] Connecting to a local Incus over a Unix socket 
DEBUG  [2024-11-14T00:12:23+01:00] Sending request to Incus                      etag= method=GET url="http://unix.socket/1.0"
DEBUG  [2024-11-14T00:12:23+01:00] Got response struct from Incus               
DEBUG  [2024-11-14T00:12:23+01:00] 
	{
		"config": {},
		"api_extensions": [
			"storage_zfs_remove_snapshots",
			"container_host_shutdown_timeout",
			"container_stop_priority",
			"container_syscall_filtering",
			"auth_pki",
			"container_last_used_at",
			"etag",
			"patch",
			"usb_devices",
			"https_allowed_credentials",
			"image_compression_algorithm",
			"directory_manipulation",
			"container_cpu_time",
			"storage_zfs_use_refquota",
			"storage_lvm_mount_options",
			"network",
			"profile_usedby",
			"container_push",
			"container_exec_recording",
			"certificate_update",
			"container_exec_signal_handling",
			"gpu_devices",
			"container_image_properties",
			"migration_progress",
			"id_map",
			"network_firewall_filtering",
			"network_routes",
			"storage",
			"file_delete",
			"file_append",
			"network_dhcp_expiry",
			"storage_lvm_vg_rename",
			"storage_lvm_thinpool_rename",
			"network_vlan",
			"image_create_aliases",
			"container_stateless_copy",
			"container_only_migration",
			"storage_zfs_clone_copy",
			"unix_device_rename",
			"storage_lvm_use_thinpool",
			"storage_rsync_bwlimit",
			"network_vxlan_interface",
			"storage_btrfs_mount_options",
			"entity_description",
			"image_force_refresh",
			"storage_lvm_lv_resizing",
			"id_map_base",
			"file_symlinks",
			"container_push_target",
			"network_vlan_physical",
			"storage_images_delete",
			"container_edit_metadata",
			"container_snapshot_stateful_migration",
			"storage_driver_ceph",
			"storage_ceph_user_name",
			"resource_limits",
			"storage_volatile_initial_source",
			"storage_ceph_force_osd_reuse",
			"storage_block_filesystem_btrfs",
			"resources",
			"kernel_limits",
			"storage_api_volume_rename",
			"network_sriov",
			"console",
			"restrict_dev_incus",
			"migration_pre_copy",
			"infiniband",
			"dev_incus_events",
			"proxy",
			"network_dhcp_gateway",
			"file_get_symlink",
			"network_leases",
			"unix_device_hotplug",
			"storage_api_local_volume_handling",
			"operation_description",
			"clustering",
			"event_lifecycle",
			"storage_api_remote_volume_handling",
			"nvidia_runtime",
			"container_mount_propagation",
			"container_backup",
			"dev_incus_images",
			"container_local_cross_pool_handling",
			"proxy_unix",
			"proxy_udp",
			"clustering_join",
			"proxy_tcp_udp_multi_port_handling",
			"network_state",
			"proxy_unix_dac_properties",
			"container_protection_delete",
			"unix_priv_drop",
			"pprof_http",
			"proxy_haproxy_protocol",
			"network_hwaddr",
			"proxy_nat",
			"network_nat_order",
			"container_full",
			"backup_compression",
			"nvidia_runtime_config",
			"storage_api_volume_snapshots",
			"storage_unmapped",
			"projects",
			"network_vxlan_ttl",
			"container_incremental_copy",
			"usb_optional_vendorid",
			"snapshot_scheduling",
			"snapshot_schedule_aliases",
			"container_copy_project",
			"clustering_server_address",
			"clustering_image_replication",
			"container_protection_shift",
			"snapshot_expiry",
			"container_backup_override_pool",
			"snapshot_expiry_creation",
			"network_leases_location",
			"resources_cpu_socket",
			"resources_gpu",
			"resources_numa",
			"kernel_features",
			"id_map_current",
			"event_location",
			"storage_api_remote_volume_snapshots",
			"network_nat_address",
			"container_nic_routes",
			"cluster_internal_copy",
			"seccomp_notify",
			"lxc_features",
			"container_nic_ipvlan",
			"network_vlan_sriov",
			"storage_cephfs",
			"container_nic_ipfilter",
			"resources_v2",
			"container_exec_user_group_cwd",
			"container_syscall_intercept",
			"container_disk_shift",
			"storage_shifted",
			"resources_infiniband",
			"daemon_storage",
			"instances",
			"image_types",
			"resources_disk_sata",
			"clustering_roles",
			"images_expiry",
			"resources_network_firmware",
			"backup_compression_algorithm",
			"ceph_data_pool_name",
			"container_syscall_intercept_mount",
			"compression_squashfs",
			"container_raw_mount",
			"container_nic_routed",
			"container_syscall_intercept_mount_fuse",
			"container_disk_ceph",
			"virtual-machines",
			"image_profiles",
			"clustering_architecture",
			"resources_disk_id",
			"storage_lvm_stripes",
			"vm_boot_priority",
			"unix_hotplug_devices",
			"api_filtering",
			"instance_nic_network",
			"clustering_sizing",
			"firewall_driver",
			"projects_limits",
			"container_syscall_intercept_hugetlbfs",
			"limits_hugepages",
			"container_nic_routed_gateway",
			"projects_restrictions",
			"custom_volume_snapshot_expiry",
			"volume_snapshot_scheduling",
			"trust_ca_certificates",
			"snapshot_disk_usage",
			"clustering_edit_roles",
			"container_nic_routed_host_address",
			"container_nic_ipvlan_gateway",
			"resources_usb_pci",
			"resources_cpu_threads_numa",
			"resources_cpu_core_die",
			"api_os",
			"container_nic_routed_host_table",
			"container_nic_ipvlan_host_table",
			"container_nic_ipvlan_mode",
			"resources_system",
			"images_push_relay",
			"network_dns_search",
			"container_nic_routed_limits",
			"instance_nic_bridged_vlan",
			"network_state_bond_bridge",
			"usedby_consistency",
			"custom_block_volumes",
			"clustering_failure_domains",
			"resources_gpu_mdev",
			"console_vga_type",
			"projects_limits_disk",
			"network_type_macvlan",
			"network_type_sriov",
			"container_syscall_intercept_bpf_devices",
			"network_type_ovn",
			"projects_networks",
			"projects_networks_restricted_uplinks",
			"custom_volume_backup",
			"backup_override_name",
			"storage_rsync_compression",
			"network_type_physical",
			"network_ovn_external_subnets",
			"network_ovn_nat",
			"network_ovn_external_routes_remove",
			"tpm_device_type",
			"storage_zfs_clone_copy_rebase",
			"gpu_mdev",
			"resources_pci_iommu",
			"resources_network_usb",
			"resources_disk_address",
			"network_physical_ovn_ingress_mode",
			"network_ovn_dhcp",
			"network_physical_routes_anycast",
			"projects_limits_instances",
			"network_state_vlan",
			"instance_nic_bridged_port_isolation",
			"instance_bulk_state_change",
			"network_gvrp",
			"instance_pool_move",
			"gpu_sriov",
			"pci_device_type",
			"storage_volume_state",
			"network_acl",
			"migration_stateful",
			"disk_state_quota",
			"storage_ceph_features",
			"projects_compression",
			"projects_images_remote_cache_expiry",
			"certificate_project",
			"network_ovn_acl",
			"projects_images_auto_update",
			"projects_restricted_cluster_target",
			"images_default_architecture",
			"network_ovn_acl_defaults",
			"gpu_mig",
			"project_usage",
			"network_bridge_acl",
			"warnings",
			"projects_restricted_backups_and_snapshots",
			"clustering_join_token",
			"clustering_description",
			"server_trusted_proxy",
			"clustering_update_cert",
			"storage_api_project",
			"server_instance_driver_operational",
			"server_supported_storage_drivers",
			"event_lifecycle_requestor_address",
			"resources_gpu_usb",
			"clustering_evacuation",
			"network_ovn_nat_address",
			"network_bgp",
			"network_forward",
			"custom_volume_refresh",
			"network_counters_errors_dropped",
			"metrics",
			"image_source_project",
			"clustering_config",
			"network_peer",
			"linux_sysctl",
			"network_dns",
			"ovn_nic_acceleration",
			"certificate_self_renewal",
			"instance_project_move",
			"storage_volume_project_move",
			"cloud_init",
			"network_dns_nat",
			"database_leader",
			"instance_all_projects",
			"clustering_groups",
			"ceph_rbd_du",
			"instance_get_full",
			"qemu_metrics",
			"gpu_mig_uuid",
			"event_project",
			"clustering_evacuation_live",
			"instance_allow_inconsistent_copy",
			"network_state_ovn",
			"storage_volume_api_filtering",
			"image_restrictions",
			"storage_zfs_export",
			"network_dns_records",
			"storage_zfs_reserve_space",
			"network_acl_log",
			"storage_zfs_blocksize",
			"metrics_cpu_seconds",
			"instance_snapshot_never",
			"certificate_token",
			"instance_nic_routed_neighbor_probe",
			"event_hub",
			"agent_nic_config",
			"projects_restricted_intercept",
			"metrics_authentication",
			"images_target_project",
			"images_all_projects",
			"cluster_migration_inconsistent_copy",
			"cluster_ovn_chassis",
			"container_syscall_intercept_sched_setscheduler",
			"storage_lvm_thinpool_metadata_size",
			"storage_volume_state_total",
			"instance_file_head",
			"instances_nic_host_name",
			"image_copy_profile",
			"container_syscall_intercept_sysinfo",
			"clustering_evacuation_mode",
			"resources_pci_vpd",
			"qemu_raw_conf",
			"storage_cephfs_fscache",
			"network_load_balancer",
			"vsock_api",
			"instance_ready_state",
			"network_bgp_holdtime",
			"storage_volumes_all_projects",
			"metrics_memory_oom_total",
			"storage_buckets",
			"storage_buckets_create_credentials",
			"metrics_cpu_effective_total",
			"projects_networks_restricted_access",
			"storage_buckets_local",
			"loki",
			"acme",
			"internal_metrics",
			"cluster_join_token_expiry",
			"remote_token_expiry",
			"init_preseed",
			"storage_volumes_created_at",
			"cpu_hotplug",
			"projects_networks_zones",
			"network_txqueuelen",
			"cluster_member_state",
			"instances_placement_scriptlet",
			"storage_pool_source_wipe",
			"zfs_block_mode",
			"instance_generation_id",
			"disk_io_cache",
			"amd_sev",
			"storage_pool_loop_resize",
			"migration_vm_live",
			"ovn_nic_nesting",
			"oidc",
			"network_ovn_l3only",
			"ovn_nic_acceleration_vdpa",
			"cluster_healing",
			"instances_state_total",
			"auth_user",
			"security_csm",
			"instances_rebuild",
			"numa_cpu_placement",
			"custom_volume_iso",
			"network_allocations",
			"zfs_delegate",
			"storage_api_remote_volume_snapshot_copy",
			"operations_get_query_all_projects",
			"metadata_configuration",
			"syslog_socket",
			"event_lifecycle_name_and_project",
			"instances_nic_limits_priority",
			"disk_initial_volume_configuration",
			"operation_wait",
			"image_restriction_privileged",
			"cluster_internal_custom_volume_copy",
			"disk_io_bus",
			"storage_cephfs_create_missing",
			"instance_move_config",
			"ovn_ssl_config",
			"certificate_description",
			"disk_io_bus_virtio_blk",
			"loki_config_instance",
			"instance_create_start",
			"clustering_evacuation_stop_options",
			"boot_host_shutdown_action",
			"agent_config_drive",
			"network_state_ovn_lr",
			"image_template_permissions",
			"storage_bucket_backup",
			"storage_lvm_cluster",
			"shared_custom_block_volumes",
			"auth_tls_jwt",
			"oidc_claim",
			"device_usb_serial",
			"numa_cpu_balanced",
			"image_restriction_nesting",
			"network_integrations",
			"instance_memory_swap_bytes",
			"network_bridge_external_create",
			"network_zones_all_projects",
			"storage_zfs_vdev",
			"container_migration_stateful",
			"profiles_all_projects",
			"instances_scriptlet_get_instances",
			"instances_scriptlet_get_cluster_members",
			"instances_scriptlet_get_project",
			"network_acl_stateless",
			"instance_state_started_at",
			"networks_all_projects",
			"network_acls_all_projects",
			"storage_buckets_all_projects",
			"resources_load",
			"instance_access",
			"project_access",
			"projects_force_delete",
			"resources_cpu_flags",
			"disk_io_bus_cache_filesystem",
			"instance_oci",
			"clustering_groups_config",
			"instances_lxcfs_per_instance",
			"clustering_groups_vm_cpu_definition",
			"disk_volume_subpath",
			"projects_limits_disk_pool",
			"network_ovn_isolated",
			"qemu_raw_qmp",
			"network_load_balancer_health_check",
			"oidc_scopes",
			"network_integrations_peer_name",
			"qemu_scriptlet",
			"instance_auto_restart",
			"storage_lvm_metadatasize",
			"ovn_nic_promiscuous",
			"ovn_nic_ip_address_none",
			"instances_state_os_info",
			"network_load_balancer_state",
			"instance_nic_macvlan_mode",
			"storage_lvm_cluster_create",
			"network_ovn_external_interfaces"
		],
		"api_status": "stable",
		"api_version": "1.0",
		"auth": "trusted",
		"public": false,
		"auth_methods": [
			"tls"
		],
		"auth_user_name": "root",
		"auth_user_method": "unix",
		"environment": {
			"addresses": [],
			"architectures": [
				"x86_64",
				"i686"
			],
			"certificate": "-----BEGIN CERTIFICATE-----\nMIICQjCCAcmgAwIBAgIRAPNK9A1pSQhAfNQz+CJrxOkwCgYIKoZIzj0EAwMwSDEZ\nMBcGA1UEChMQTGludXggQ29udGFpbmVyczErMCkGA1UEAwwicm9vdEBjaS1hcHBz\nLXRlbXBsYXRlLnl1bm9ob3N0Lm9yZzAeFw0yNDExMDYxNjQxMTJaFw0zNDExMDQx\nNjQxMTJaMEgxGTAXBgNVBAoTEExpbnV4IENvbnRhaW5lcnMxKzApBgNVBAMMInJv\nb3RAY2ktYXBwcy10ZW1wbGF0ZS55dW5vaG9zdC5vcmcwdjAQBgcqhkjOPQIBBgUr\ngQQAIgNiAAQyJAW4NyvK9l2J6ViVxk48+W6f2H8McLJ911vvBHRfJQIYiggIqoCM\n/qK2LqEa+37wBl2DHKs2mgwscNcCc3AEbWFzHByJ6Ug1+Cyo8/ta1e10QUBlu8af\nxQDsfrO+T3GjdzB1MA4GA1UdDwEB/wQEAwIFoDATBgNVHSUEDDAKBggrBgEFBQcD\nATAMBgNVHRMBAf8EAjAAMEAGA1UdEQQ5MDeCHWNpLWFwcHMtdGVtcGxhdGUueXVu\nb2hvc3Qub3JnhwR/AAABhxAAAAAAAAAAAAAAAAAAAAABMAoGCCqGSM49BAMDA2cA\nMGQCMHUxSRquKEDmpeEREtg39IzlIrW8CLypKaf+m21Jagf06MnUw3xumEExDPY7\nWoDQTAIwb4/t8Je5FtecnnmsOwEbB8JFuOks2OrBNNasi4jKnPV/DLvYK1VO+JF7\nYoIsC9p8\n-----END CERTIFICATE-----\n",
			"certificate_fingerprint": "b5ecab9e9501758aed58618d0926744ff9870d0c44b8cc892c64dbbc378885b9",
			"driver": "lxc",
			"driver_version": "6.0.2",
			"firewall": "nftables",
			"kernel": "Linux",
			"kernel_architecture": "x86_64",
			"kernel_features": {
				"idmapped_mounts": "true",
				"netnsid_getifaddrs": "true",
				"seccomp_listener": "true",
				"seccomp_listener_continue": "true",
				"uevent_injection": "true",
				"unpriv_binfmt": "false",
				"unpriv_fscaps": "true"
			},
			"kernel_version": "6.1.0-26-amd64",
			"lxc_features": {
				"cgroup2": "true",
				"core_scheduling": "true",
				"devpts_fd": "true",
				"idmapped_mounts_v2": "true",
				"mount_injection_file": "true",
				"network_gateway_device_route": "true",
				"network_ipvlan": "true",
				"network_l2proxy": "true",
				"network_phys_macvlan_mtu": "true",
				"network_veth_router": "true",
				"pidfd": "true",
				"seccomp_allow_deny_syntax": "true",
				"seccomp_notify": "true",
				"seccomp_proxy_send_notify_fd": "true"
			},
			"os_name": "Debian GNU/Linux",
			"os_version": "12",
			"project": "default",
			"server": "incus",
			"server_clustered": false,
			"server_event_mode": "full-mesh",
			"server_name": "ci-apps-trixie.yunohost.org",
			"server_pid": 1939726,
			"server_version": "6.6",
			"storage": "btrfs",
			"storage_version": "6.2",
			"storage_supported_drivers": [
				{
					"Name": "dir",
					"Version": "1",
					"Remote": false
				},
				{
					"Name": "lvm",
					"Version": "2.03.16(2) (2022-05-18) / 1.02.185 (2022-05-18) / 4.47.0",
					"Remote": false
				},
				{
					"Name": "lvmcluster",
					"Version": "2.03.16(2) (2022-05-18) / 1.02.185 (2022-05-18) / 4.47.0",
					"Remote": true
				},
				{
					"Name": "btrfs",
					"Version": "6.2",
					"Remote": false
				}
			]
		}
	} 
DEBUG  [2024-11-14T00:12:23+01:00] Connected to the websocket: ws://unix.socket/1.0/events 
DEBUG  [2024-11-14T00:12:23+01:00] Sending request to Incus                      etag= method=POST url="http://unix.socket/1.0/images"
DEBUG  [2024-11-14T00:12:23+01:00] 
	{
		"auto_update": true,
		"properties": null,
		"public": false,
		"expires_at": "0001-01-01T00:00:00Z",
		"profiles": null,
		"filename": "",
		"source": {
			"alias": "",
			"certificate": "",
			"protocol": "simplestreams",
			"server": "https://repo.yunohost.org/incus",
			"image_type": "",
			"mode": "pull",
			"type": "image",
			"url": "",
			"name": "",
			"fingerprint": "yunohost/trixie-unstable/appci",
			"secret": "",
			"project": ""
		},
		"compression_algorithm": "",
		"aliases": null
	} 
DEBUG  [2024-11-14T00:12:23+01:00] Got operation from Incus                     
DEBUG  [2024-11-14T00:12:23+01:00] 
	{
		"id": "8a8365f8-e3eb-46aa-bee4-9475b245a558",
		"class": "task",
		"description": "Downloading image",
		"created_at": "2024-11-14T00:12:23.453309982+01:00",
		"updated_at": "2024-11-14T00:12:23.453309982+01:00",
		"status": "Running",
		"status_code": 103,
		"resources": {},
		"metadata": null,
		"may_cancel": false,
		"err": "",
		"location": "none"
	} 
DEBUG  [2024-11-14T00:12:23+01:00] Sending request to Incus                      etag= method=GET url="http://unix.socket/1.0/operations/8a8365f8-e3eb-46aa-bee4-9475b245a558"
DEBUG  [2024-11-14T00:12:23+01:00] Got response struct from Incus               
DEBUG  [2024-11-14T00:12:23+01:00] 
	{
		"id": "8a8365f8-e3eb-46aa-bee4-9475b245a558",
		"class": "task",
		"description": "Downloading image",
		"created_at": "2024-11-14T00:12:23.453309982+01:00",
		"updated_at": "2024-11-14T00:12:23.453309982+01:00",
		"status": "Running",
		"status_code": 103,
		"resources": {},
		"metadata": null,
		"may_cancel": false,
		"err": "",
		"location": "none"
	} 
Error: Failed remote image download: Unable to fetch https://repo.yunohost.org/incus/images/1b46a0dfacf7eedb0bfef3f9a3877b54ed9185c8960de67e1b53d2e0e8d52f91.incus_combined.tar.gz: 404 Not Found

It looks like the incus daemon caches the index of the repository but doesn't know when to refresh it. When reading the logs of the nginx server serving the simplestreams repository, i can see the requests for the image, but not for the images.json index.
You can see the sha1 mismatch (500dedd061a0 when listing, 1b46a0dfacf7eedb0bfef3f9a3877b54ed9185c8960de67e1b53d2e0e8d52f91 when fetching)

(I'm going to sleep, hence the very short issue, but feel free to request more info, i'll reply tomorrow :D )

[stgraber]

The Incus daemon caches simplestreams data for up to an hour.
The client on the other hand only caches for 5 minutes.

So incus image list remote: would hit a 5 minutes client cache at ~/.cache/incus/.
On the server side, the equivalent cache is an hour long and stores the files at /var/cache/incus/.

Can you confirm that you are past that 1h cache expiry in your case?

[Salamandar]

Ah ! Thank you ! Yes, indeed, you're right. I was clearing the client cache hoping it would help… but could not find the server cache.

IMHO, when hitting 404s, the cache should be invalidated.