From b3fb2785894237b4199172e971a4ed57fd4eb249 Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Mon, 17 Jun 2024 11:56:53 +0200 Subject: [PATCH] Breaking change: Remove LMDB-to-safe-mode migration --- CHANGELOG.md | 4 + Cargo.lock | 30 -- DEPENDENCIES.md | 214 ----------- glean-core/Cargo.toml | 2 +- .../android-native/dependency-licenses.xml | 9 - glean-core/android/dependency-licenses.xml | 9 - glean-core/src/database/mod.rs | 350 +----------------- supply-chain/config.toml | 4 - supply-chain/imports.lock | 19 - 9 files changed, 6 insertions(+), 635 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f1fb892d20..17667f14a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ [Full changelog](https://github.com/mozilla/glean/compare/v61.2.0...main) +* General + * **BREAKING**: Remove LMDB-to-safe-mode migration. + Safe-mode became the default in Glean v51. ([#TODO]()) + # v61.2.0 (2024-10-07) [Full changelog](https://github.com/mozilla/glean/compare/v61.1.0...v61.2.0) diff --git a/Cargo.lock b/Cargo.lock index ac3f0aa302..c3a9179164 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -523,29 +523,6 @@ version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" -[[package]] -name = "lmdb-rkv" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "447a296f7aca299cfbb50f4e4f3d49451549af655fb7215d7f8c0c3d64bad42b" -dependencies = [ - "bitflags 1.3.2", - "byteorder", - "libc", - "lmdb-rkv-sys", -] - -[[package]] -name = "lmdb-rkv-sys" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61b9ce6b3be08acefa3003c57b7565377432a89ec24476bbe72e11d101f852fe" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - [[package]] name = "log" version = "0.4.20" @@ -666,12 +643,6 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" -[[package]] -name = "pkg-config" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" - [[package]] name = "plain" version = "0.2.3" @@ -732,7 +703,6 @@ dependencies = [ "byteorder", "id-arena", "lazy_static", - "lmdb-rkv", "log", "ordered-float", "paste", diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md index ca4be60292..c5aea6cc21 100644 --- a/DEPENDENCIES.md +++ b/DEPENDENCIES.md @@ -203,218 +203,6 @@ The following text applies to code linked from these dependencies: The following text applies to code linked from these dependencies: -* [lmdb-rkv 0.14.0]( https://github.com/mozilla/lmdb-rs.git ) - -``` - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2014 Dan Burkert - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -``` -## Apache License 2.0 - - -The following text applies to code linked from these dependencies: - * [rkv 0.19.0]( https://github.com/mozilla/rkv ) * [static_assertions 1.1.0]( https://github.com/nvzqz/static-assertions-rs ) @@ -2298,7 +2086,6 @@ limitations under the License. The following text applies to code linked from these dependencies: -* [bitflags 1.3.2]( https://github.com/bitflags/bitflags ) * [bitflags 2.4.1]( https://github.com/bitflags/bitflags ) * [camino 1.1.4]( https://github.com/camino-rs/camino ) * [cfg-if 1.0.0]( https://github.com/alexcrichton/cfg-if ) @@ -4023,7 +3810,6 @@ limitations under the License. The following text applies to code linked from these dependencies: -* [lmdb-rkv-sys 0.11.2]( https://github.com/mozilla/lmdb-rs.git ) * [siphasher 0.3.10]( https://github.com/jedisct1/rust-siphash ) * [tinyvec_macros 0.1.0]( https://github.com/Soveu/tinyvec_macros ) * [winapi-x86_64-pc-windows-gnu 0.4.0]( https://github.com/retep998/winapi-rs ) diff --git a/glean-core/Cargo.toml b/glean-core/Cargo.toml index 1b46816bb0..8393ce3ed8 100644 --- a/glean-core/Cargo.toml +++ b/glean-core/Cargo.toml @@ -30,7 +30,7 @@ maintenance = { status = "actively-developed" } [dependencies] serde = { version = "1.0.104", features = ["derive"] } serde_json = "1.0.44" -rkv = { version = "0.19.0", default-features = false, features = ["lmdb"] } +rkv = { version = "0.19.0", default-features = false } bincode = "1.2.1" log = "0.4.8" uuid = { version = "1.0", features = ["v4"] } diff --git a/glean-core/android-native/dependency-licenses.xml b/glean-core/android-native/dependency-licenses.xml index c29816bc79..87b20360e1 100644 --- a/glean-core/android-native/dependency-licenses.xml +++ b/glean-core/android-native/dependency-licenses.xml @@ -6,9 +6,6 @@ the details of which are reproduced below. --> Apache License 2.0: miniz_oxide https://github.com/Frommi/miniz_oxide/tree/master/miniz_oxide - - Apache License 2.0: lmdb-rkv - https://github.com/mozilla/lmdb-rs.git Apache License 2.0: rkv https://github.com/mozilla/rkv @@ -90,9 +87,6 @@ the details of which are reproduced below. Apache License 2.0: bitflags https://github.com/bitflags/bitflags - - Apache License 2.0: bitflags - https://github.com/bitflags/bitflags Apache License 2.0: camino https://github.com/camino-rs/camino @@ -222,9 +216,6 @@ the details of which are reproduced below. Apache License 2.0: android_logger https://github.com/Nercury/android_logger-rs - - Apache License 2.0: lmdb-rkv-sys - https://github.com/mozilla/lmdb-rs.git Apache License 2.0: siphasher https://github.com/jedisct1/rust-siphash diff --git a/glean-core/android/dependency-licenses.xml b/glean-core/android/dependency-licenses.xml index c29816bc79..87b20360e1 100644 --- a/glean-core/android/dependency-licenses.xml +++ b/glean-core/android/dependency-licenses.xml @@ -6,9 +6,6 @@ the details of which are reproduced below. --> Apache License 2.0: miniz_oxide https://github.com/Frommi/miniz_oxide/tree/master/miniz_oxide - - Apache License 2.0: lmdb-rkv - https://github.com/mozilla/lmdb-rs.git Apache License 2.0: rkv https://github.com/mozilla/rkv @@ -90,9 +87,6 @@ the details of which are reproduced below. Apache License 2.0: bitflags https://github.com/bitflags/bitflags - - Apache License 2.0: bitflags - https://github.com/bitflags/bitflags Apache License 2.0: camino https://github.com/camino-rs/camino @@ -222,9 +216,6 @@ the details of which are reproduced below. Apache License 2.0: android_logger https://github.com/Nercury/android_logger-rs - - Apache License 2.0: lmdb-rkv-sys - https://github.com/mozilla/lmdb-rs.git Apache License 2.0: siphasher https://github.com/jedisct1/rust-siphash diff --git a/glean-core/src/database/mod.rs b/glean-core/src/database/mod.rs index 20cb0f151e..398137db3f 100644 --- a/glean-core/src/database/mod.rs +++ b/glean-core/src/database/mod.rs @@ -16,8 +16,7 @@ use std::time::{Duration, Instant}; use crate::ErrorKind; -use rkv::migrator::Migrator; -use rkv::{MigrateError, StoreError, StoreOptions}; +use rkv::{StoreError, StoreOptions}; /// Unwrap a `Result`s `Ok` value or do the specified action. /// @@ -89,94 +88,6 @@ pub fn rkv_new(path: &Path) -> std::result::Result<(Rkv, RkvLoadState), rkv::Sto } } -fn delete_and_log(path: &Path, msg: &str) { - if let Err(err) = fs::remove_file(path) { - match err.kind() { - std::io::ErrorKind::NotFound => { - // Silently drop this error, the file was already non-existing. - } - _ => log::warn!("{}", msg), - } - } -} - -fn delete_lmdb_database(path: &Path) { - let datamdb = path.join("data.mdb"); - delete_and_log(&datamdb, "Failed to delete old data."); - - let lockmdb = path.join("lock.mdb"); - delete_and_log(&lockmdb, "Failed to delete old lock."); -} - -/// Migrate from LMDB storage to safe-mode storage. -/// -/// This migrates the data once, then deletes the LMDB storage. -/// The safe-mode storage must be empty for it to work. -/// Existing data will not be overwritten. -/// If the destination database is not empty the LMDB database is deleted -/// without migrating data. -/// This is a no-op if no LMDB database file exists. -pub fn migrate(path: &Path, dst_env: &Rkv) { - log::debug!("Migrating files in {}", path.display()); - - // Shortcut if no data to migrate is around. - let datamdb = path.join("data.mdb"); - if !datamdb.exists() { - log::debug!("No data to migrate."); - return; - } - - // We're handling the same error cases as `easy_migrate_lmdb_to_safe_mode`, - // but annotate each why they don't cause problems for Glean. - // Additionally for known cases we delete the LMDB database regardless. - let should_delete = - match Migrator::open_and_migrate_lmdb_to_safe_mode(path, |builder| builder, dst_env) { - // Source environment is corrupted. - // We start fresh with the new database. - Err(MigrateError::StoreError(StoreError::FileInvalid)) => true, - Err(MigrateError::StoreError(StoreError::DatabaseCorrupted)) => true, - // Path not accessible. - // Somehow our directory vanished between us creating it and reading from it. - // Nothing we can do really. - Err(MigrateError::StoreError(StoreError::IoError(_))) => true, - // Path accessible but incompatible for configuration. - // This should not happen, we never used storages that safe-mode doesn't understand. - // If it does happen, let's start fresh and use the safe-mode from now on. - Err(MigrateError::StoreError(StoreError::UnsuitableEnvironmentPath(_))) => true, - // Nothing to migrate. - // Source database was empty. We just start fresh anyway. - Err(MigrateError::SourceEmpty) => true, - // Migrating would overwrite. - // Either a previous migration failed and we still started writing data, - // or someone placed back an old data file. - // In any case we better stay on the new data and delete the old one. - Err(MigrateError::DestinationNotEmpty) => { - log::warn!("Failed to migrate old data. Destination was not empty"); - true - } - // An internal lock was poisoned. - // This would only happen if multiple things run concurrently and one crashes. - Err(MigrateError::ManagerPoisonError) => false, - // Couldn't close source environment and delete files on disk (e.g. other stores still open). - // This could only happen if multiple instances are running, - // we leave files in place. - Err(MigrateError::CloseError(_)) => false, - // Other store errors are never returned from the migrator. - // We need to handle them to please rustc. - Err(MigrateError::StoreError(_)) => false, - // Other errors can't happen, so this leaves us with the Ok case. - // This already deleted the LMDB files. - Ok(()) => false, - }; - - if should_delete { - log::debug!("Need to delete remaining LMDB files."); - delete_lmdb_database(path); - } - - log::debug!("Migration ended. Safe-mode database in {}", path.display()); -} - use crate::common_metric_data::CommonMetricDataInternal; use crate::metrics::Metric; use crate::Glean; @@ -356,7 +267,6 @@ impl Database { fs::create_dir_all(path)?; let (rkv, load_state) = rkv_new(path)?; - migrate(path, &rkv); log::info!("Database initialized"); Ok((rkv, load_state)) @@ -1613,7 +1523,6 @@ mod test { use std::fs::File; use super::*; - use rkv::Value; #[test] fn empty_data_file() { @@ -1657,262 +1566,5 @@ mod test { "Load error recorded" ); } - - #[test] - fn migration_works_on_startup() { - let dir = tempdir().unwrap(); - - let database_dir = dir.path().join("db"); - let datamdb = database_dir.join("data.mdb"); - let lockmdb = database_dir.join("lock.mdb"); - let safebin = database_dir.join("data.safe.bin"); - - assert!(!safebin.exists()); - assert!(!datamdb.exists()); - assert!(!lockmdb.exists()); - - let store_name = "store1"; - let metric_name = "bool"; - let key = Database::get_storage_key(store_name, Some(metric_name)); - - // Ensure some old data in the LMDB format exists. - { - fs::create_dir_all(&database_dir).expect("create dir"); - let rkv_db = rkv::Rkv::new::(&database_dir).expect("rkv env"); - - let store = rkv_db - .open_single("ping", StoreOptions::create()) - .expect("opened"); - let mut writer = rkv_db.write().expect("writer"); - let metric = Metric::Boolean(true); - let value = bincode::serialize(&metric).expect("serialized"); - store - .put(&mut writer, &key, &Value::Blob(&value)) - .expect("wrote"); - writer.commit().expect("committed"); - - assert!(datamdb.exists()); - assert!(lockmdb.exists()); - assert!(!safebin.exists()); - } - - // First open should migrate the data. - { - let db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); - let safebin = database_dir.join("data.safe.bin"); - assert!(safebin.exists(), "safe-mode file should exist"); - assert!(!datamdb.exists(), "LMDB data should be deleted"); - assert!(!lockmdb.exists(), "LMDB lock should be deleted"); - - let mut stored_metrics = vec![]; - let mut snapshotter = |name: &[u8], metric: &Metric| { - let name = str::from_utf8(name).unwrap().to_string(); - stored_metrics.push((name, metric.clone())) - }; - db.iter_store_from(Lifetime::Ping, "store1", None, &mut snapshotter); - - assert_eq!(1, stored_metrics.len()); - assert_eq!(metric_name, stored_metrics[0].0); - assert_eq!(&Metric::Boolean(true), &stored_metrics[0].1); - } - - // Next open should not re-create the LMDB files. - { - let db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); - let safebin = database_dir.join("data.safe.bin"); - assert!(safebin.exists(), "safe-mode file exists"); - assert!(!datamdb.exists(), "LMDB data should not be recreated"); - assert!(!lockmdb.exists(), "LMDB lock should not be recreated"); - - let mut stored_metrics = vec![]; - let mut snapshotter = |name: &[u8], metric: &Metric| { - let name = str::from_utf8(name).unwrap().to_string(); - stored_metrics.push((name, metric.clone())) - }; - db.iter_store_from(Lifetime::Ping, "store1", None, &mut snapshotter); - - assert_eq!(1, stored_metrics.len()); - assert_eq!(metric_name, stored_metrics[0].0); - assert_eq!(&Metric::Boolean(true), &stored_metrics[0].1); - } - } - - #[test] - fn migration_doesnt_overwrite() { - let dir = tempdir().unwrap(); - - let database_dir = dir.path().join("db"); - let datamdb = database_dir.join("data.mdb"); - let lockmdb = database_dir.join("lock.mdb"); - let safebin = database_dir.join("data.safe.bin"); - - assert!(!safebin.exists()); - assert!(!datamdb.exists()); - assert!(!lockmdb.exists()); - - let store_name = "store1"; - let metric_name = "counter"; - let key = Database::get_storage_key(store_name, Some(metric_name)); - - // Ensure some old data in the LMDB format exists. - { - fs::create_dir_all(&database_dir).expect("create dir"); - let rkv_db = rkv::Rkv::new::(&database_dir).expect("rkv env"); - - let store = rkv_db - .open_single("ping", StoreOptions::create()) - .expect("opened"); - let mut writer = rkv_db.write().expect("writer"); - let metric = Metric::Counter(734); // this value will be ignored - let value = bincode::serialize(&metric).expect("serialized"); - store - .put(&mut writer, &key, &Value::Blob(&value)) - .expect("wrote"); - writer.commit().expect("committed"); - - assert!(datamdb.exists()); - assert!(lockmdb.exists()); - } - - // Ensure some data exists in the new database. - { - fs::create_dir_all(&database_dir).expect("create dir"); - let rkv_db = - rkv::Rkv::new::(&database_dir).expect("rkv env"); - - let store = rkv_db - .open_single("ping", StoreOptions::create()) - .expect("opened"); - let mut writer = rkv_db.write().expect("writer"); - let metric = Metric::Counter(2); - let value = bincode::serialize(&metric).expect("serialized"); - store - .put(&mut writer, &key, &Value::Blob(&value)) - .expect("wrote"); - writer.commit().expect("committed"); - - assert!(safebin.exists()); - } - - // First open should try migration and ignore it, because destination is not empty. - // It also deletes the leftover LMDB database. - { - let db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); - let safebin = database_dir.join("data.safe.bin"); - assert!(safebin.exists(), "safe-mode file should exist"); - assert!(!datamdb.exists(), "LMDB data should be deleted"); - assert!(!lockmdb.exists(), "LMDB lock should be deleted"); - - let mut stored_metrics = vec![]; - let mut snapshotter = |name: &[u8], metric: &Metric| { - let name = str::from_utf8(name).unwrap().to_string(); - stored_metrics.push((name, metric.clone())) - }; - db.iter_store_from(Lifetime::Ping, "store1", None, &mut snapshotter); - - assert_eq!(1, stored_metrics.len()); - assert_eq!(metric_name, stored_metrics[0].0); - assert_eq!(&Metric::Counter(2), &stored_metrics[0].1); - } - } - - #[test] - fn migration_ignores_broken_database() { - let dir = tempdir().unwrap(); - - let database_dir = dir.path().join("db"); - let datamdb = database_dir.join("data.mdb"); - let lockmdb = database_dir.join("lock.mdb"); - let safebin = database_dir.join("data.safe.bin"); - - assert!(!safebin.exists()); - assert!(!datamdb.exists()); - assert!(!lockmdb.exists()); - - let store_name = "store1"; - let metric_name = "counter"; - let key = Database::get_storage_key(store_name, Some(metric_name)); - - // Ensure some old data in the LMDB format exists. - { - fs::create_dir_all(&database_dir).expect("create dir"); - fs::write(&datamdb, "bogus").expect("dbfile created"); - - assert!(datamdb.exists()); - } - - // Ensure some data exists in the new database. - { - fs::create_dir_all(&database_dir).expect("create dir"); - let rkv_db = - rkv::Rkv::new::(&database_dir).expect("rkv env"); - - let store = rkv_db - .open_single("ping", StoreOptions::create()) - .expect("opened"); - let mut writer = rkv_db.write().expect("writer"); - let metric = Metric::Counter(2); - let value = bincode::serialize(&metric).expect("serialized"); - store - .put(&mut writer, &key, &Value::Blob(&value)) - .expect("wrote"); - writer.commit().expect("committed"); - } - - // First open should try migration and ignore it, because destination is not empty. - // It also deletes the leftover LMDB database. - { - let db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); - let safebin = database_dir.join("data.safe.bin"); - assert!(safebin.exists(), "safe-mode file should exist"); - assert!(!datamdb.exists(), "LMDB data should be deleted"); - assert!(!lockmdb.exists(), "LMDB lock should be deleted"); - - let mut stored_metrics = vec![]; - let mut snapshotter = |name: &[u8], metric: &Metric| { - let name = str::from_utf8(name).unwrap().to_string(); - stored_metrics.push((name, metric.clone())) - }; - db.iter_store_from(Lifetime::Ping, "store1", None, &mut snapshotter); - - assert_eq!(1, stored_metrics.len()); - assert_eq!(metric_name, stored_metrics[0].0); - assert_eq!(&Metric::Counter(2), &stored_metrics[0].1); - } - } - - #[test] - fn migration_ignores_empty_database() { - let dir = tempdir().unwrap(); - - let database_dir = dir.path().join("db"); - let datamdb = database_dir.join("data.mdb"); - let lockmdb = database_dir.join("lock.mdb"); - let safebin = database_dir.join("data.safe.bin"); - - assert!(!safebin.exists()); - assert!(!datamdb.exists()); - assert!(!lockmdb.exists()); - - // Ensure old LMDB database exists, but is empty. - { - fs::create_dir_all(&database_dir).expect("create dir"); - let rkv_db = rkv::Rkv::new::(&database_dir).expect("rkv env"); - drop(rkv_db); - assert!(datamdb.exists()); - assert!(lockmdb.exists()); - } - - // First open should try migration, but find no data. - // safe-mode does not write an empty database to disk. - // It also deletes the leftover LMDB database. - { - let _db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); - let safebin = database_dir.join("data.safe.bin"); - assert!(!safebin.exists(), "safe-mode file should exist"); - assert!(!datamdb.exists(), "LMDB data should be deleted"); - assert!(!lockmdb.exists(), "LMDB lock should be deleted"); - } - } } } diff --git a/supply-chain/config.toml b/supply-chain/config.toml index 43605ce16d..7440f9da2f 100644 --- a/supply-chain/config.toml +++ b/supply-chain/config.toml @@ -124,10 +124,6 @@ criteria = "safe-to-run" version = "0.2.139" criteria = "safe-to-deploy" -[[exemptions.lmdb-rkv-sys]] -version = "0.11.2" -criteria = "safe-to-deploy" - [[exemptions.memchr]] version = "2.5.0" criteria = "safe-to-deploy" diff --git a/supply-chain/imports.lock b/supply-chain/imports.lock index ca2dbcecdc..951b0573cd 100644 --- a/supply-chain/imports.lock +++ b/supply-chain/imports.lock @@ -261,12 +261,6 @@ a few `unsafe` blocks related to utf-8 validation which are locally verifiable as correct and otherwise this crate is good to go. """ -[[audits.bytecode-alliance.audits.pkg-config]] -who = "Pat Hickey " -criteria = "safe-to-deploy" -version = "0.3.25" -notes = "This crate shells out to the pkg-config executable, but it appears to sanitize inputs reasonably." - [[audits.bytecode-alliance.audits.proc-macro2]] who = "Pat Hickey " criteria = "safe-to-deploy" @@ -690,13 +684,6 @@ criteria = "safe-to-deploy" delta = "0.2.147 -> 0.2.148" aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" -[[audits.mozilla.audits.lmdb-rkv]] -who = "Bobby Holley " -criteria = "safe-to-deploy" -version = "0.14.0" -notes = "Victor and Myk developed this crate at Mozilla." -aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" - [[audits.mozilla.audits.log]] who = "Mike Hommey " criteria = "safe-to-deploy" @@ -717,12 +704,6 @@ version = "0.2.15" notes = "All code written or reviewed by Josh Stone." aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" -[[audits.mozilla.audits.pkg-config]] -who = "Mike Hommey " -criteria = "safe-to-deploy" -delta = "0.3.25 -> 0.3.26" -aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" - [[audits.mozilla.audits.proc-macro2]] who = "Nika Layzell " criteria = "safe-to-deploy"