Add more crates

- data-pdf
- data-error
- data-json
- data-link
- fs-storage
- fs-atomic-light
- data-json (ex fs-utils)
- data-error (ex fs-utils)

Cargo.toml

@@ -1,16 +1,25 @@
 [workspace]
 members = [
+    "data-error",
+    "data-json",
+    "data-link",
+    "data-pdf",
     "data-resource",
     "fs-atomic-versions",
+    "fs-atomic-light",
     "fs-index",
-    "fs-utils"
+    "fs-storage",
 ]
 
 default-members = [
+    "data-error",
+    "data-json",
+    "data-link",
+    "data-pdf",
     "data-resource",
     "fs-atomic-versions",
     "fs-index",
-    "fs-utils",
+    "fs-storage",
 ]
 
 resolver = "2"

README.md

@@ -12,13 +12,15 @@ The purpose of the library is to manage _resource index_ of folders with various
 
 <div align="center">
 
-| Package              | Description                                |
-| -------------------- | ------------------------------------------ |
-| `data-resource`      | Resource hashing and ID construction       |
-| `fs-index`           | Resource Index construction and updating   |
-| `fs-atomic-light`    | Temp file-based preventing of dirty writes |
-| `fs-atomic-versions` | Version-based preventing of dirty writes   |
-| `fs-utils`           | Utility functions and common code          |
+| Package         | Description                              |
+| --------------- | ---------------------------------------- |
+| `data-resource` | Resource hashing and ID construction     |
+| `fs-index`      | Resource Index construction and updating |
+| `fs-storage`    | Filesystem storage for resources         |
+| `data-link`     | Linking resources                        |
+| `data-pdf`      | PDF handling                             |
+| `data-error`    | Error handling                           |
+| `data-json`     | JSON serialization and deserialization   |
 
 </div>
 

fs-utils/Cargo.toml → data-error/Cargo.toml

@@ -1,16 +1,11 @@
 [package]
-name = "fs-utils"
+name = "data-error"
 version = "0.1.0"
 edition = "2021"
 
-
 [dependencies]
 thiserror = "1"
 reqwest = "0.11.11"
 serde_json = "1.0.82"
 anyhow = "1"
 url = { version = "2.2.2", features = ["serde"] }
-
-[dev-dependencies]
-rstest = "0.18"
-tempdir = "0.3"

fs-utils/src/errors.rs → data-error/src/lib.rs

@@ -1,5 +1,4 @@
 use std::str::Utf8Error;
-
 use thiserror::Error;
 
 pub type Result<T> = std::result::Result<T, ArklibError>;

data-json/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "data-json"
+version = "0.1.0"
+edition = "2021"
+
+
+[dependencies]
+serde_json = "1.0.82"
+
+[dev-dependencies]
+rstest = "0.18"

data-link/Cargo.toml

@@ -0,0 +1,27 @@
+[package]
+name = "data-link"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+name = "data_link"
+crate-type = ["rlib"]
+bench = false
+
+[dependencies]
+fs-atomic-light = { path = "../fs-atomic-light" }
+fs-atomic-versions = { path = "../fs-atomic-versions" }
+fs-storage = { path = "../fs-storage" }
+data-resource = { path = "../data-resource" }
+data-error = { path = "../data-error" }
+
+log = { version = "0.4.17", features = ["release_max_level_off"] }
+serde_json = "1.0.82"
+serde = { version = "1.0.138", features = ["derive"] }
+url = { version = "2.2.2", features = ["serde"] }
+reqwest = "0.11.11"
+scraper = "0.13.0"
+tokio = { version = "1", features = ["full"] }
+
+[dev-dependencies]
+tempdir = "0.3.7"

fs-index/src/link.rs → data-link/src/lib.rs

@@ -1,10 +1,12 @@
-use crate::storage::meta::store_metadata;
-use crate::storage::prop::store_properties;
-use crate::{
-    storage::prop::load_raw_properties, AtomicFile, Result, ARK_FOLDER,
-    PREVIEWS_STORAGE_FOLDER, PROPERTIES_STORAGE_FOLDER,
-};
+use data_error::Result;
 use data_resource::ResourceId;
+use fs_atomic_versions::atomic::AtomicFile;
+use fs_storage::meta::store_metadata;
+use fs_storage::prop::load_raw_properties;
+use fs_storage::prop::store_properties;
+use fs_storage::{
+    ARK_FOLDER, PREVIEWS_STORAGE_FOLDER, PROPERTIES_STORAGE_FOLDER,
+};
 use reqwest::header::HeaderValue;
 use scraper::{Html, Selector};
 use serde::{Deserialize, Serialize};
@@ -25,20 +27,6 @@ pub struct Properties {
     pub title: String,
     pub desc: Option<String>,
 }
-/// Write data to a tempory file and move that written file to destination
-///
-/// May failed if writing or moving failed
-fn temp_and_move(
-    data: &[u8],
-    dest_dir: impl AsRef<Path>,
-    filename: &str,
-) -> Result<()> {
-    let mut path = std::env::temp_dir();
-    path.push(filename);
-    std::fs::write(&path, data)?;
-    std::fs::copy(path, dest_dir.as_ref().join(filename))?;
-    Ok(())
-}
 
 impl Link {
     pub fn new(url: Url, title: String, desc: Option<String>) -> Self {
@@ -62,6 +50,7 @@ impl Link {
             .join(PROPERTIES_STORAGE_FOLDER)
             .join(id.to_string());
         let file = AtomicFile::new(path)?;
+
         let current = file.load()?;
         let data = current.read_to_string()?;
         let user_meta: Properties = serde_json::from_str(&data)?;
@@ -103,7 +92,7 @@ impl Link {
 
         // Resources are stored in the folder chosen by user
         let bytes = self.url.as_str().as_bytes();
-        temp_and_move(bytes, root.as_ref(), &id_string)?;
+        fs_atomic_light::temp_and_move(bytes, root.as_ref(), &id_string)?;
         //User defined properties
         store_properties(&root, id, &self.prop)?;
 
@@ -194,6 +183,7 @@ fn select_og(html: &Html, tag: OpenGraphTag) -> Option<String> {
 
     None
 }
+
 fn select_desc(html: &Html) -> Option<String> {
     let selector = Selector::parse("meta[name=\"description\"]").unwrap();
 
@@ -205,6 +195,7 @@ fn select_desc(html: &Html) -> Option<String> {
 
     None
 }
+
 fn select_title(html: &Html) -> Option<String> {
     let selector = Selector::parse("title").unwrap();
     if let Some(element) = html.select(&selector).next() {
@@ -213,6 +204,7 @@ fn select_title(html: &Html) -> Option<String> {
 
     None
 }
+
 #[derive(Debug, Serialize, Deserialize, Default, Clone)]
 pub struct OpenGraph {
     /// Represents the "og:title" OpenGraph meta tag.
@@ -234,6 +226,7 @@ pub struct OpenGraph {
     /// Represents the "og:locale" OpenGraph meta tag
     locale: Option<String>,
 }
+
 impl OpenGraph {
     pub async fn fetch_image(&self) -> Option<Vec<u8>> {
         if let Some(url) = &self.image {

data-pdf/Cargo.toml

@@ -0,0 +1,28 @@
+[package]
+name = "data-pdf"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+name = "data_pdf"
+crate-type = ["rlib"]
+bench = false
+
+[dependencies]
+once_cell = "1.16.0"
+image = "0.24.2"
+pdfium-render = { git = "https://github.com/ajrcarey/pdfium-render", rev = "d2559c1", features = [
+    "thread_safe",
+    "sync",
+] }
+
+[dev-dependencies]
+tempdir = "0.3.7"
+
+[build-dependencies]
+flate2 = "1.0.24"
+fs_extra = "1.2.0"
+tar = "0.4.38"
+target-lexicon = "0.12.4"
+ureq = "2.4.0"
+ring = "=0.17.5"

data-resource/Cargo.toml

@@ -6,7 +6,7 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-fs-utils = { path = "../fs-utils" }
+data-error = { path = "../data-error" }
 fs-atomic-versions = { path = "../fs-atomic-versions" }
 
 log = { version = "0.4.17", features = ["release_max_level_off"] }

data-resource/src/lib.rs

@@ -9,7 +9,7 @@ use std::io::{BufRead, BufReader};
 use std::path::Path;
 use std::str::FromStr;
 
-use fs_utils::errors::{ArklibError, Result};
+use data_error::{ArklibError, Result};
 
 #[derive(
     Eq,

fs-atomic-light/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "fs-atomic-light"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+name = "fs_atomic_light"
+crate-type = ["rlib"]
+bench = false
+
+[dependencies]
+data-error = { path = "../data-error" }

fs-atomic-light/src/lib.rs

@@ -0,0 +1,23 @@
+use data_error::Result;
+
+use std::env;
+use std::fs;
+use std::path::Path;
+use std::str;
+
+/// Write data to a tempory file and move that written file to destination
+///
+/// May failed if writing or moving failed
+pub fn temp_and_move(
+    data: &[u8],
+    dest_dir: impl AsRef<Path>,
+    filename: &str,
+) -> Result<()> {
+    let mut path = env::temp_dir();
+    path.push(filename);
+
+    fs::write(&path, data)?;
+    fs::copy(path, dest_dir.as_ref().join(filename))?;
+
+    Ok(())
+}

fs-atomic-versions/Cargo.toml

@@ -9,7 +9,7 @@ crate-type = ["rlib"]
 bench = false
 
 [dependencies]
-fs-utils = { path = "../fs-utils" }
+data-error = { path = "../data-error" }
 
 log = { version = "0.4.17", features = ["release_max_level_off"] }
 anyhow = "1.0.58"

fs-atomic-versions/src/app_id.rs

@@ -2,7 +2,7 @@ use std::{fs, path::Path};
 
 use anyhow::anyhow;
 
-use fs_utils::errors::{ArklibError, Result};
+use data_error::{ArklibError, Result};
 
 use crate::{APP_ID_FILE, APP_ID_PATH};
 

fs-atomic-versions/src/atomic/file.rs

@@ -103,7 +103,7 @@ fn parse_version(filename: Option<&str>) -> Option<usize> {
 }
 
 impl AtomicFile {
-    pub fn new(path: impl Into<PathBuf>) -> fs_utils::errors::Result<Self> {
+    pub fn new(path: impl Into<PathBuf>) -> data_error::Result<Self> {
         let directory = path.into();
         // This UID must be treated as confidential information.
         // Depending on network transport used to sync the files (if any),

fs-index/Cargo.toml

@@ -9,8 +9,9 @@ crate-type = ["rlib"]
 bench = false
 
 [dependencies]
-fs-utils = { path = "../fs-utils" }
+data-error = { path = "../data-error" }
 fs-atomic-versions = { path = "../fs-atomic-versions" }
+fs-storage = { path = "../fs-storage" }
 data-resource = { path = "../data-resource" }
 
 log = { version = "0.4.17", features = ["release_max_level_off"] }
@@ -19,20 +20,7 @@ anyhow = "1.0.58"
 lazy_static = "1.4.0"
 canonical-path = "2.0.2"
 pathdiff = "0.2.1"
-serde_json = "1.0.82"
-serde = { version = "1.0.138", features = ["derive"] }
-url = { version = "2.2.2", features = ["serde"] }
-reqwest = "0.11.11"
-scraper = "0.13.0"
-tokio = { version = "1", features = ["full"] }
 itertools = "0.10.5"
-once_cell = "1.16.0"
-image = "0.24.2"
-pdfium-render = { git = "https://github.com/ajrcarey/pdfium-render", rev = "d2559c1", features = [
-    "thread_safe",
-    "sync",
-] }
-
 
 [dev-dependencies]
 uuid = { version = "1.6.1", features = ["v4"] }
@@ -43,14 +31,6 @@ criterion = { version = "0.5", features = ["html_reports"] }
 pprof = { version = "0.13", features = ["criterion", "flamegraph"] }
 rand = "0.8"
 
-[build-dependencies]
-flate2 = "1.0.24"
-fs_extra = "1.2.0"
-tar = "0.4.38"
-target-lexicon = "0.12.4"
-ureq = "2.4.0"
-ring = "=0.17.5"
-
 [[bench]]
 name = "index_build_benchmark"
 harness = false