Start using permalinks and add support for the schema date

zspell-index-cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "cargo-index-cli"
-version = "0.4.0"
+version = "0.5.0"
 edition = "2021"
 license = "Apache-2.0"
 description = "CLI to generate a ZSpell index"
@@ -13,4 +13,4 @@ serde = { version = "1.0.195", features = ["derive"] }
 serde_json = "1.0.111"
 ureq = { version = "2.9.1", features = ["json"] }
 uuid = { version = "1.6.1", features = ["serde", "v7"] }
-zspell-index = { path = "../zspell-index", version = "0.4" }
+zspell-index = { path = "../zspell-index", version = "0.5" }

zspell-index-cli/src/main.rs

@@ -3,21 +3,24 @@
 
 use anyhow::{bail, Context};
 use serde::Deserialize;
+use serde_json::Value;
 use std::{env, fs, path::Path, time::Duration};
 use zspell_index::{DictionaryFormat, Downloadable, Index, IndexEntry};
 
-const WOOORM_ROOT_URL: &str =
-    "https://api.github.com/repos/wooorm/dictionaries/contents/dictionaries";
+const WOOORM_API_URL: &str = "https://api.github.com/repos/wooorm/dictionaries";
+const WOOORM_BRANCH_NAME: &str = "main";
 const WOOORM_TAG: &str = "source-wooorm";
 const APP_USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION"),);
-const OUTPUT_DIR: &str = env!("CARGO_MANIFEST_DIR");
+const OUTPUT_DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/..");
 const FILE_NAME: &str = "zspell-index.json";
 const FILE_NAME_PRETTY: &str = "zspell-index-pretty.json";
 
 /// Contents of a directory
 #[derive(Debug, Deserialize)]
 struct Tree(Vec<Listing>);
 
+// FIXME: use permalinks
+/// A single subdirectory or file within a [`Tree`]
 #[derive(Debug, Deserialize)]
 #[allow(dead_code)]
 struct Listing {
@@ -50,6 +53,7 @@ fn make_client() -> ureq::Agent {
         let with_header = if let Ok(var) = env::var("GITHUB_API_TOKEN") {
             req.set("Authorization", &format!("Bearer {var}"))
         } else {
+            eprintln!("tip: set the GITHUB_API_TOKEN environment variable to avoid rate limiting");
             req
         };
 
@@ -108,23 +112,64 @@ fn update_inner(
         is_ext: false,
         id: uuid::Uuid::now_v7(),
         format: DictionaryFormat::Hunspell {
-            afx: make_downloadable(afx_entry)?,
+            aff: make_downloadable(afx_entry)?,
             dic: make_downloadable(dic_entry)?,
         },
         lic: make_downloadable(lic_entry)?,
     };
     Ok(Some(ret))
 }
 
+fn get_latest_hash(agent: &ureq::Agent) -> anyhow::Result<Box<str>> {
+    let resp: Value = agent
+        .get(&format!(
+            "{WOOORM_API_URL}/commits/{WOOORM_BRANCH_NAME}?per_page=1"
+        ))
+        .call()
+        .context("requesting latest git hash")?
+        .into_json()?;
+
+    let Value::Object(mut map) = resp else {
+        bail!("invalid response");
+    };
+
+    let Some(Value::String(sha)) = map.remove("sha") else {
+        bail!("response is missing sha");
+    };
+
+    eprintln!("using git hash {sha}");
+    Ok(sha.into())
+}
+
+fn write_to_file(
+    index: &Index,
+    output_path: &Path,
+    output_path_pretty: &Path,
+) -> anyhow::Result<()> {
+    let ser = serde_json::to_string(index)?;
+    let ser_pretty = serde_json::to_string_pretty(index)?;
+
+    eprintln!("writing output to {}", output_path.display());
+    fs::write(output_path, ser)?;
+    eprintln!("writing pretty output to {}", output_path_pretty.display());
+    fs::write(output_path_pretty, ser_pretty)?;
+    Ok(())
+}
+
 fn update_from_wooorm() -> anyhow::Result<()> {
     let agent = make_client();
+    let git_ref = get_latest_hash(&agent)?;
     let all_langs: Tree = agent
-        .get(WOOORM_ROOT_URL)
+        .get(&format!(
+            "{WOOORM_API_URL}/contents/dictionaries?ref={git_ref}"
+        ))
         .call()
         .context("requesting root listing")?
         .into_json()?;
 
     let mut items = Vec::new();
+    let mut has_errors = false;
+
     for dir in all_langs.0.iter() {
         let lang = &dir.name;
         let ListingContents::Dir = dir.contents else {
@@ -133,24 +178,30 @@ fn update_from_wooorm() -> anyhow::Result<()> {
 
         eprintln!("locating dictionary {lang}");
 
-        let item = update_inner(lang, &dir.url, &agent)?;
-        let Some(item) = item else { continue };
-
-        items.push(item);
+        match update_inner(lang, &dir.url, &agent) {
+            Ok(Some(item)) => items.push(item),
+            Ok(None) => continue,
+            Err(e) => {
+                eprintln!("error with {lang}: {e}. skipping");
+                has_errors = true;
+                continue;
+            }
+        }
     }
 
     let mut index = Index::new();
     index.items = items.into_boxed_slice();
 
-    let output_path = Path::new(OUTPUT_DIR).join(FILE_NAME);
-    let output_path_pretty = Path::new(OUTPUT_DIR).join(FILE_NAME_PRETTY);
-    let ser = serde_json::to_string(&index)?;
-    let ser_pretty = serde_json::to_string_pretty(&index)?;
+    let mut output_path = Path::new(OUTPUT_DIR).join(FILE_NAME);
+    let mut output_path_pretty = Path::new(OUTPUT_DIR).join(FILE_NAME_PRETTY);
 
-    eprintln!("writing output to {}", output_path.display());
-    fs::write(output_path, ser)?;
-    eprintln!("writing pretty output to {}", output_path_pretty.display());
-    fs::write(output_path_pretty, ser_pretty)?;
+    if has_errors {
+        eprintln!("errors encountered during update. writing incomplete files.");
+        output_path.set_extension("incomplete.json");
+        output_path_pretty.set_extension("incomplete.json");
+    }
+
+    write_to_file(&index, &output_path, &output_path_pretty)?;
 
     Ok(())
 }