diff --git a/public/ARC-isa-cwl-decorations.png b/public/ARC-isa-cwl-decorations.png new file mode 100644 index 0000000..0ef0e07 Binary files /dev/null and b/public/ARC-isa-cwl-decorations.png differ diff --git a/public/arc-bag-your-research.png b/public/arc-bag-your-research.png new file mode 100644 index 0000000..88c05a8 Binary files /dev/null and b/public/arc-bag-your-research.png differ diff --git a/public/arc-process-graph.png b/public/arc-process-graph.png new file mode 100644 index 0000000..fdfef66 Binary files /dev/null and b/public/arc-process-graph.png differ diff --git a/public/arc-ro-crate-profiles.png b/public/arc-ro-crate-profiles.png new file mode 100644 index 0000000..1490f2a Binary files /dev/null and b/public/arc-ro-crate-profiles.png differ diff --git a/public/arc-ro-crate.png b/public/arc-ro-crate.png new file mode 100644 index 0000000..d36de3c Binary files /dev/null and b/public/arc-ro-crate.png differ diff --git a/public/documentation-principle-assay.png b/public/documentation-principle-assay.png new file mode 100644 index 0000000..ed47e1d Binary files /dev/null and b/public/documentation-principle-assay.png differ diff --git a/public/documentation-principle-run.png b/public/documentation-principle-run.png new file mode 100644 index 0000000..8921e09 Binary files /dev/null and b/public/documentation-principle-run.png differ diff --git a/public/documentation-principle-study.png b/public/documentation-principle-study.png new file mode 100644 index 0000000..4a734b9 Binary files /dev/null and b/public/documentation-principle-study.png differ diff --git a/public/documentation-principle-workflow.png b/public/documentation-principle-workflow.png new file mode 100644 index 0000000..46e4de5 Binary files /dev/null and b/public/documentation-principle-workflow.png differ diff --git a/public/fairness-gradual-process.png b/public/fairness-gradual-process.png new file mode 100644 index 0000000..e7a880c Binary files /dev/null and b/public/fairness-gradual-process.png differ diff --git a/public/fairness-long-tail-data.png b/public/fairness-long-tail-data.png new file mode 100644 index 0000000..81748f3 Binary files /dev/null and b/public/fairness-long-tail-data.png differ diff --git a/public/orga-principle-folder2process.png b/public/orga-principle-folder2process.png new file mode 100644 index 0000000..7bb831c Binary files /dev/null and b/public/orga-principle-folder2process.png differ diff --git a/public/orga-principle-scaffold.png b/public/orga-principle-scaffold.png new file mode 100644 index 0000000..17595d6 Binary files /dev/null and b/public/orga-principle-scaffold.png differ diff --git a/public/research-data-lifecycle.png b/public/research-data-lifecycle.png new file mode 100644 index 0000000..0b7151c Binary files /dev/null and b/public/research-data-lifecycle.png differ diff --git a/src/components/Home/CommunityCards.astro b/src/components/Home/CommunityCards.astro index 770d49c..d2c875e 100644 --- a/src/components/Home/CommunityCards.astro +++ b/src/components/Home/CommunityCards.astro @@ -14,17 +14,17 @@ import Card from '../Card.astro' diff --git a/src/components/Home/DeveloperCards.astro b/src/components/Home/DeveloperCards.astro index c299cd0..3e316f6 100644 --- a/src/components/Home/DeveloperCards.astro +++ b/src/components/Home/DeveloperCards.astro @@ -1,4 +1,5 @@ --- +import { URLS } from "../../statics"; import Card from '../Card.astro' --- @@ -12,19 +13,19 @@ import Card from '../Card.astro'
diff --git a/src/components/Home/RDMGraphNavigation.astro b/src/components/Home/RDMGraphNavigation.astro index ad8a7a2..3420dc3 100644 --- a/src/components/Home/RDMGraphNavigation.astro +++ b/src/components/Home/RDMGraphNavigation.astro @@ -4,7 +4,7 @@ import GraphNavigation from "../GraphNavigation.astro"; import { Color } from "../GraphNavigation.astro"; const circles = [ - { id: 1, cx: 8, cy: 25, r: 5, href: '#link4', text: 'FAIR Digital Object', angle: 20 }, + { id: 1, cx: 8, cy: 25, r: 5, href: URLS.Internal_Home + "/details/arc-fdo", text: 'FAIR Digital Object', angle: 20 }, { id: 2, cx: 30, cy: 40, r: 5, href: '#link4', text: 'validation', angle: 180 }, { id: 3, cx: 60, cy: 8, r: 5, href: '#link3', text: 'continuous Integration', angle: 80 }, { id: 4, cx: 90, cy: 40, r: 5, href: '#link2', text: 'versioning', angle: 250 }, diff --git a/src/components/Home/ResearchGraphNavigation.astro b/src/components/Home/ResearchGraphNavigation.astro index 22c6293..02a581c 100644 --- a/src/components/Home/ResearchGraphNavigation.astro +++ b/src/components/Home/ResearchGraphNavigation.astro @@ -5,10 +5,10 @@ import { Color } from "../GraphNavigation.astro"; const circles = [ { id: 1, cx: 8, cy: 10, r: 5, href: URLS.Internal_Home + "/details/documentation-principle", text: 'documentation principle', angle: 80 }, - { id: 2, cx: 30, cy: 40, r: 5, href: '#link2', text: 'organization principle', angle: 200 }, + { id: 2, cx: 30, cy: 40, r: 5, href: URLS.Internal_Home + "/details/organization-principle", text: 'organization principle', angle: 200 }, { id: 3, cx: 60, cy: 35, r: 5, href: '#link3', text: 'quality control', angle: 130 }, { id: 4, cx: 90, cy: 15, r: 5, href: '#link4', text: 'exchange & publication', angle: 10 }, - { id: 5, cx: 140, cy: 25, r: 5, href: '#link4', text: 'RDM & FAIRness', angle: 210 }, + { id: 5, cx: 140, cy: 25, r: 5, href: URLS.Internal_Home + "/details/fairness-and-rdm", text: 'RDM & FAIRness', angle: 210 }, ]; --- diff --git a/src/pages/details/arc-data-model.md b/src/pages/details/arc-data-model.md new file mode 100644 index 0000000..b057117 --- /dev/null +++ b/src/pages/details/arc-data-model.md @@ -0,0 +1,26 @@ +--- +layout: ../../layouts/MarkdownLayout.astro +title: 'ARC data model' +pubDate: 2024-09-13 +description: 'A short description of the ARC data mode.' +author: 'Timo Mühlhaus' +image: + url: 'https://docs.astro.build/assets/rose.webp' + alt: 'The Astro logo on a dark background with a pink glow.' +tags: ["RO-Crate", "FAIR digital object","JSON-LD"] +--- + +ARC is an implementation of a FAIR Digital Object (FDO), utilizing RO-Crate with Schema.org and Bioschemas objects as its foundation, and further enhancing it with additional metadata and structure. An RO-Crate serves as a research object, composed of a collection of research elements and data, enabling detailed descriptions of these collections. Research elements in this context include samples, measurement data, and other research outputs. + +![ARC RO Crate](/arc-website/arc-ro-crate.png) + +ARC extends the basic RO-Crate concept by incorporating detailed descriptions of the processes that lead to the generation of data. This enhancement allows the data model to represent a complete process graph, encompassing experimental procedures, simulations, analyses, and the interconnections and provenance among them. +In this model, research elements are the nodes of the process graph, while the connections between them, defined as lab processes, are represented by edges. Each process can be further specified and annotated with explanatory and descriptive metadata using lists of PropertyValues, enhancing its clarity and traceability. + +![ARC RO Crate](/arc-website/ARC-isa-cwl-decorations.png) + +To specialize ARC for biological data, the widely recognized ISA model (Investigation, Study, Assay) is employed, alongside the abstract Common Workflow Language (CWL) for workflows. The ISA model and CWL provide additional layers of metadata and structure, allowing for more precise definitions of processes and data. Dataset objects within ARC can implement either ISA or CWL interface object definitions, ensuring compatibility and standardization across various biological datasets and workflows. + +![ARC RO Crate](/arc-website/arc-ro-crate-profiles.png) + +This approach elevates ARC from merely documenting research objects to providing a full, interconnected representation of the research process, from data generation to analysis, making it a powerful tool for tracing and reproducing scientific investigations. diff --git a/src/pages/details/arc-fdo.md b/src/pages/details/arc-fdo.md new file mode 100644 index 0000000..ca9aa9e --- /dev/null +++ b/src/pages/details/arc-fdo.md @@ -0,0 +1,59 @@ +--- +layout: ../../layouts/MarkdownLayout.astro +title: 'ARCs are FDOs' +pubDate: 2024-09-13 +description: 'ARCs RO-crates are FDOs.' +author: 'Timo Mühlhaus' +image: + url: 'https://docs.astro.build/assets/rose.webp' + alt: 'The Astro logo on a dark background with a pink glow.' +tags: ["tools", "services", "community"] +--- + +Enabling the researchers to achieve data FAIRness and facilitating FAIR compliance is central to ARC concept. FAIRness is a continuum that evolves and can be continuously improved. This philosophy is encapsulated in ARC's "Immutable yet Evolving" design, which allows for the dynamic addition of metadata through collaborative metadata evolution. This approach accommodates the diverse and changing requirements of researchers from various domains, supported by a Git-based versioning mechanism and a comprehensive suite of tools and services. +ARCs employ FAIRness by design and provide dual representations: a Git file and directory project structure that is also represented as an RO Crate JSON-LD using the ARC-ISA RO Crate profile. This consistently integrates existing standards and, combined with Git version tracking, achieves full compliance with the FAIR Data Maturity Model of the RDA. By leveraging standardized metadata and ontology-driven annotation supported by annotation tools, ARC ensures that research data is increasingly FAIR. Interoperability is emphasized by improving the integration of ARC FDOs in workflows, search engines, analytic tools, community platforms, databases, endpoint repositories, and journals. + +The RDA has proposed the so-called FAIR Data Maturity Indicators to gauge the degree to which FAIR principles are implemented. The following table shows these indicators for ARCs to be FAIR. For each indicator, +- Priority (E – Essential; I – Important; U – Useful), +- Readiness (C – Complete; P – Planned) of implementation +- Description of realization by ARC +are listed. + +| ID | P | R | Description | +|----------------|-----|-----|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| RDA-F1-01D | | | | +| RDA-F1-01M | E | E | | +| RDA-F1-02D | E | C | Unique and persistent identifiers are assigned for data and metadata by Git project-ID, URL, index and version hash. | +| RDA-F1-02M | E | C | Standardized Git data projects including necessary and auxiliary metadata (ARCs) can be directly published as ARC data publications using an Invenio instance. Stable DOIs are provided for both the underlying Git repository as well as specific published versions. | +| RDA-A1-03M | E | E | | +| RDA-A1-03M | C | C | Metadata development is achieved using the DataPLANT Broker Ontology as a stable index, with participation and involvement of the national and international plant research communities. Community participation is actively enabled via schema templates (available openly on GitHub) to close the ontology gap and provide flexibility to quickly support vocabulary that describes novel and individual experiments. Metadata can also be reused from existing ontologies. | +| RDA-F2-01M | E | C | ARCs define a structure that combines ISA and CWL metadata information into a set of fields. Metadata values are provided by applying relevant and standard schemata and ontologies. Pre-provided templates facilitate and accelerate the addition of metadata descriptions across common cases (e.g., research question or institute policy). | +| RDA-F3-01M | E | C | ARCs contain administrative metadata within the ISA investigation. Combined with Git project-ID, URL, index and version hash an identifier is provided for all data. | +| RDA-F4-01M | E | C | ARCs are de-facto RO-Crate31 (JSON-LD) implementations with content following the ISA and/or CWL standard. Compliant crates with appropriate descriptions (using schema.org terms and the bioschema.org profile description) can be generated, either automatically without user intervention using the PLANTdataHUB’s continuous integration produces, or triggered manually, using a prototypical implementation. Such crates are comprehensible to data harvesters such as Google data search, the FAIDARE data discovery portal, and DataPLANT’s PLANTdataHUB ARC registry. | +| RDA-A1-04M | | | | +| RDA-A1-04D | | | | +| RDA-A1.1-01M | | | | +| RDA-A1.1-01D | E | E | | +| RDA-A1-02M | I | C | Git repository access is based on standardized protocols such as HTTPS and SSH; these can be used for retrieving all data and metadata. | +| RDA-A1-02D | E | C | Git is supported by a multitude of easily accessible desktop and web tools available for management. DataPLANT provides a custom user interface aimed at plant biologists. | +| RDA-A1-01M | I | C | In combination with Git and Invenio, each ARC contains the metadata information needed to enable users to access its data. | +| RDA-A1.2-01D | U | C | Data and metadata can be made accessible both publicly and securely via open authentication mechanisms proxied by DataPLANT keycloak for unification. | +| RDA-A2-01M | | | | +| RDA-A1-03D | E | C | DataPLANT’s data publication process triggers metadata harvesting and persistent storage in an Invenio instance, providing an individual DOI. | +| RDA-I1-02M | E | C | (Meta)data use a formal, accessible, shared, and broadly applicable language for knowledge representation. The tabular ISA facilitates maintenance by humans, whereas JSON-LD-based RO-Crate31 that contains the complete ISA JSON as well as the CWL information completes a formal machine-actionability. In order to bridge the gap between user interaction and machine-actionability, RO-Crate ARC decoration is produced by a continuous integration process either triggered manually or after synchronization with the PLANTdataHUB. | +| RDA-I1-01D | | | | +| RDA-I2-01D | I | C | For data and derived data representation, the use of technology-specific yet vendor-agnostic file formats is suggested in the ARC. This is ideal to cover standard processing steps in a FAIR manner, but predefined formats fall short in the description of individual analyses. Therefore, DataPLANT uses an additional, ISA-based data map layer to annotate result file content (e.g., a data frame derived from an analysis). | +| RDA-I2-01M | I | C | All user input is supported by the use of curated FAIR-compliant ontologies that are schematized (RDF/XML, OBO, OWL) and find- and identifiable via PID (purl). | +| RDA-I3-01M | E | C | Using FAIR vocabularies in the context of the ISA framework, all (meta)data are referenced, including a reference to the ontology used in the investigation section. | +| RDA-I3-02D | E | C | In the RO-Crate representation of ARCs, all data included in the investigation are referenced; moreover, Git partial checkout and referencing mechanism allows automatic tracking of references to reused or linked data. | +| RDA-I1-02M | | | | +| RDA-I3-02M | | | | +| RDA-I3-04M | U | C | Machine-understandability is achieved by the JSON-LD-based RO-Crate ARC decoration, including references to internal and external data within the metadata. | +| RDA-I3-03M | I | C | DataPLANT’s Broker Ontology provides stable indexing via Git. Furthermore, individual terms are linked and cross-referenced to other ontologies and can be shadowed by PIDs of the target ontology after the brokering process for compliance. | +| RDA-R1-01M | E | C | Experimental processes are formally described using the ISA framework, while computational workflows including processing, analysis and simulations are described using CWL. While any available workflow engine/language (e.g., Galaxy8, Nextflow47, Snakemake48) can be used, the processing graph with input and output needs to be specified in CWL to ensure a homogenized interoperability and entry point for computation. | +| RDA-R1.1-01M | | | | +| RDA-R1.1-02M | | | | +| RDA-R1.1-03M | E | C | Strong orientation in the area of open-source software: Licensing options are firmly anchored in repositories and directly visible in and included in the RO-Crate. Our strong commitment to open science is reflected in the firmly established licensing options within repositories, directly visible in and integrated into the RO-Crate. | +| RDA-R1.2-01M | I | C | By using Git, consistent versioning allows traceability (provenance) of all data according to (cross)-community-specific standards. | +| RDA-R1.3-01M | E | C | Due to the immutable yet evolving nature of an ARC especially the information completeness can be achieved gradually without a roadblock during data contextualization. This means that the annotation process never fails but will be marked as incomplete. To insure (meta)data compliance with community standard requirements a prepared as ISA-templates that can be directly used for data annotation. | +| RDA-R1.3-02M | I | C | ARC environment ensures immediate machine readability and actionability and is given its standards “AI ready”, while the git repository-based template generation crowd sources community-driven metadata standard development. | diff --git a/src/pages/details/documentation-principle.md b/src/pages/details/documentation-principle.md index e8d6287..620dc8a 100644 --- a/src/pages/details/documentation-principle.md +++ b/src/pages/details/documentation-principle.md @@ -1,13 +1,13 @@ --- layout: ../../layouts/MarkdownLayout.astro -title: 'Documentation Principle ' +title: 'Documentation and Annotation' pubDate: 2024-09-13 -description: 'A short summary for ARC related tools and services.' +description: 'A introduction to the ARC documentation and annotation principles.' author: 'Timo Mühlhaus' image: url: 'https://docs.astro.build/assets/rose.webp' alt: 'The Astro logo on a dark background with a pink glow.' -tags: ["tools", "services", "community"] +tags: ["community", "organization", "project management"] --- ## Documentation Principle @@ -19,15 +19,22 @@ To facilitate a shared understanding, a standardized nomenclature is introduced. A single ARC encapsulates one investigation. The metadata of an investigation includes a title and a description of its focus. The authors and contributors to the research may include colleagues, measurement facility experts, bioinformaticians, supervisors, and supporting technicians. Crucially, the investigation also holds information about the objects or samples under study, which may be grouped into one or more studies. The study metadata describe the samples and observables, especially the processes leading to sample generation. One or more studies may be included in an investigation, depending on how the samples are intended to be processed or analyzed. +![Documentation Principle](/arc-website/documentation-principle-study.png) -Samples, once created, typically undergo assays, which involve measurements or analyses of samples from one or multiple studies. The assay includes both the data generated and the metadata describing the process. If the samples require special treatment for a specific assay, the relevant metadata should be incorporated into the assay documentation. +Samples, once created, typically undergo assays, which involve measurements or analyses of samples from one or multiple studies. The assay includes both the data generated and the metadata describing the process. If the samples require special treatment for a specific assay, the relevant metadata should be incorporated into the assay documentation. +![Documentation Principle](/arc-website/documentation-principle-assay.png) -When computational analysis is performed on a sample or on the data resulting from an assay, this process is referred to as a run. A workflow, on the other hand, is the computational protocol detailing how the data is processed, simulated, or analyzed on a computer without actually executing the computation. Since workflows offer significant value for reuse in other datasets, they are documented separately from runs. +When computational analysis is performed on a sample or on the data resulting from an assay, this process is referred to as a run. +![Documentation Principle](/arc-website/documentation-principle-run.png) +A workflow, on the other hand, is the computational protocol detailing how the data is processed, simulated, or analyzed on a computer without actually executing the computation. Since workflows offer significant value for reuse in other datasets, they are documented separately from runs. +![Documentation Principle](/arc-website/documentation-principle-workflow.png) -Notice: The ARC is designed to document the entire journey (process) from the object of study, through measurements and analysis (as processes), to the final results. This journey represents a process of processes, capturing each stage as part of the broader transformation from observable phenomena to conclusive outcomes. The ARC annotation principle is to add tags on these process for documentation. +> Notice: The ARC is designed to document the entire journey (process) from the object of study, through measurements and analysis (as processes) to the final results. This journey represents a process of processes, capturing each stage as part of the broader transformation from observable phenomena to conclusive outcomes. The ARC annotation principle is to add tags on these process for documentation. (The term "experiment" is avoided here to prevent confusion, as it can intuitively overlap with "investigation," "study," or "assay" depending on context.) +![Documentation Principle](/arc-website/arc-process-graph.png) + ## Annotation Principle The ARC annotation principle aims to document all processes—from the object of study, through measurements and analysis, to the final results—as data. Each step, whether it involves sampling, sample preparation, measurement, simulation, or analysis, is treated as a process that generates outputs. These outputs can take the form of result files or sample identifiers, which in turn serve as inputs for subsequent processes. This allows for the chaining and branching of different processes, effectively modeling real-world workflows in the lab and providing a clear, documented path leading to the final results. @@ -44,8 +51,8 @@ Special header keys have specific meanings, such as sample name, protocol refere Following the ISA model, keys are enclosed in square brackets. Additional qualifiers may be used to further specify the key. Common qualifiers include: -- Parameter: Typically used for process-related metadata. -- Component: Refers to an element used during the process. -- Characteristic: Describes the properties or characteristics of the input to a given process. +- **Parameter:** Typically used for process-related metadata. +- **Component:** Refers to an element used during the process. +- **Characteristic:** Describes the properties or characteristics of the input to a given process. These conventions ensure a structured and consistent approach to annotating complex experimental workflows, making the data more traceable and understandable. \ No newline at end of file diff --git a/src/pages/details/fairness-and-rdm.md b/src/pages/details/fairness-and-rdm.md new file mode 100644 index 0000000..abbb875 --- /dev/null +++ b/src/pages/details/fairness-and-rdm.md @@ -0,0 +1,87 @@ +--- +layout: ../../layouts/MarkdownLayout.astro +title: 'RDM and FAIRness' +pubDate: 2024-09-13 +description: 'A introduction to FAIRness.' +author: 'Timo Mühlhaus' +image: + url: 'https://docs.astro.build/assets/rose.webp' + alt: 'The Astro logo on a dark background with a pink glow.' +tags: ["Data publication", "FAIRness", "FAIR digital object"] +--- + +### Research Data Management (RDM) + +The ARC is designed to fully support Research Data Management (RDM) practices and facilitate participation in the evolving RDM ecosystem. RDM involves managing research data at every stage of its lifecycle, including: + +- Collecting +- Organizing +- Documenting +- Processing +- Analyzing +- Storing +- Sharing + +By integrating ARC into research workflows, researchers can effectively manage data across these stages, ensuring their research is well-organized, accessible, and reusable. + +![Research data lifecycle](/arc-website/research-data-lifecycle.png) + +#### Goals of Research Data Management (RDM): +- **Enhancing research effectiveness and efficiency**: Well-organized data accelerates the research process by making data easier to find, analyze, and reuse. +- **Improving data security**: Proper management helps to avoid data or metadata loss, ensuring that research outcomes are preserved over time. +- **Preventing errors**: Clear documentation and structuring minimize the risk of mistakes during data handling and analysis. +- **Increasing research impact**: High-quality data management ensures that datasets are shareable and reusable, boosting the visibility and impact of research. +- **Ensuring reproducibility**: Robust RDM practices make it easier to reproduce studies, which is critical for scientific credibility. +- **Promoting data reusability**: Properly managed data can be reused by other researchers, increasing its value beyond the original project. + +### Open Data: The Path to Open Science + +Publicly funded research is often expected to provide a return to society, and research data is frequently one of the most valuable outputs of these projects. Data reusability plays a crucial role in maximizing this return. + +Sharing both research data and experimental protocols is essential for ensuring reproducibility. Journals and peer review processes should emphasize the requirement for sharing this data, reinforcing transparency in scientific endeavors. Open data not only supports reproducibility but also accelerates scientific discovery by making information accessible to a wider community. + +### Data Publication: Solving the Data Reuse Problem + +Traditionally, researchers are evaluated based on the knowledge they publish in scientific journals. However, the data behind these publications also hold substantial value. + +- **Reusability**: Data can be reused to generate further insights, validate findings, or support new research directions. +- **Re-examination**: As new techniques or theories emerge, existing data may need to be revisited. +- **Evolution of conclusions**: Scientific conclusions can change or evolve when additional data is considered. + +Unfortunately, in traditional publications, data often remain buried in supplemental materials or summarized in figures, making it difficult to reuse. This approach does not scale well with the growing volumes of research data, which needs to be findable, accessible, and usable in more structured ways. + +### Data as a Primary Product in Science + +Data itself is a primary product of scientific research, holding significant intrinsic value. It can be: +- **Reused** to drive new discoveries and insights +- **Re-examined** when novel techniques or theories arise +- **Reintegrated** with new datasets to refine or revise conclusions + +However, data must be treated as atomic facts—core pieces of information that can stand alone or combine with other datasets to offer new knowledge. When managed correctly, data transcends its role in a single project, becoming a long-term asset for the broader scientific community. + +### FAIR Digital Objects (FDOs) and Data Publication + +The concept of **Fair Digital Objects (FDOs)** is becoming central to data publication. Classical publications are beginning to reference FDOs as standalone data publications, enabling data to be shared independently of traditional articles. + +The **FAIR Data Principles** (Findable, Accessible, Interoperable, Reusable) outline how data packages should be structured to ensure they are useful for both humans and machines. Adhering to these principles is increasingly required by funding agencies and is essential for making research data valuable and sustainable. + +- **Findable**: Data should be easy to locate, indexed, and searchable. +- **Accessible**: Data should be retrievable through open, standardized protocols. +- **Interoperable**: Data should be compatible with other datasets, enabling integration. +- **Reusable**: Data should be well-documented, structured, and licensed for reuse. + +Producing FDOs that meet FAIR standards is always worthwhile. FAIRness is a continuum, and researchers can incrementally improve their data packages towards greater compliance. The ARC embraces this philosophy by supporting continuous improvement toward FAIRness. + +### FAIRness as an Ongoing Process + +FAIRness can be progressively enhanced over time. It represents a balance between the agile development of research data and the need for scholarly stability. The ARC supports this philosophy by providing researchers with the tools and structure to improve their data’s FAIRness gradually, ensuring maximum value for both the individual researcher and the broader scientific community. + +![Gradual FAIRness](/arc-website/fairness-gradual-process.png) + +### "Small Data Done Right" is Big Data + +When data are well-managed and shared effectively, even small datasets can contribute to large-scale scientific efforts. Combining data across studies requires data-sharing practices that extend beyond the lifespan of individual projects. This reinforces the idea that **research done right is teamwork by default**, and well-structured, shareable data is a cornerstone of collaborative science. + +![RDM long tail data](/arc-website/fairness-long-tail-data.png) + +By embracing these principles, ARC fosters a research environment that enhances RDM practices, promotes open science, and supports the continuous improvement of data FAIRness, driving the scientific community forward. \ No newline at end of file diff --git a/src/pages/details/organization-principle.md b/src/pages/details/organization-principle.md new file mode 100644 index 0000000..a715052 --- /dev/null +++ b/src/pages/details/organization-principle.md @@ -0,0 +1,58 @@ +--- +layout: ../../layouts/MarkdownLayout.astro +title: 'Organization and structure' +pubDate: 2024-09-13 +description: 'A introduction to the ARC organization principles.' +author: 'Timo Mühlhaus' +image: + url: 'https://docs.astro.build/assets/rose.webp' + alt: 'The Astro logo on a dark background with a pink glow.' +tags: ["community", "organization", "project management"] +--- + +### Organization Principle + +The core principle of the ARC is that collected data are stored in directories, while metadata are maintained in accompanying tables that reference and describe the data. This organizational structure is closely aligned with the ISA model, while also incorporating workflows, computational processing, and analysis results. + +![ARC scaffold structure](/arc-website/orga-principle-folder2process.png) + +The foundational idea behind ARC is to provide a directory scaffold that ensures research data, along with their processing and analysis, are organized in a structured and annotated manner. This scaffold supplies a basic file structure for organizing research locally on a personal machine, as well as on data-producing devices such as measurement instruments or compute servers. A key feature of this system is its ability to seamlessly transfer to the cloud, specifically to a DataHUB instance (e.g., Git-LFS) hosted by an institution or NFDI consortium. + +Through GIT’s versioning mechanism, data can be easily backed up, integrated across devices, and shared with collaborators. Each interaction is tracked and can be reverted if necessary. Additionally, DataHUB offers project management tools, such as task assignments and discussion boards. + +The unified structure of ARC ensures that research can be shared and understood easily by others. Several software tools are available to help create ARCs and support data analysis functionalities. While ARC requires a specific directory structure to be recognized, researchers have the flexibility to add additional files or folders as needed. + +### ARC Directory Structure + +ARC represents an entire investigation. At the top level, it includes directories named **“studies,” “assays,” “workflows,”** and **“runs,”** along with an **investigation metadata table** that holds all administrative metadata. + +![ARC scaffold structure](/arc-website/orga-principle-scaffold.png) + +#### Study +The **studies** folder contains one or more studies, each in its own directory. Each study folder contains: +- **Study metadata file**: A table with metadata describing the study. +- **Resources folder**: Contains external data used in the study. +- **Protocols folder**: Stores protocols that describe the process from starting material (or data) to samples. These protocols should be stored in a format that can be referenced in the metadata table. + +#### Assay +The **assays** folder contains one or more assays, each in its own directory. Each assay folder contains: +- **Assay metadata file**: A table with metadata describing the measurement process. +- **Dataset folder**: Holds the resulting data from the assay process, typically raw measurement files (open file formats are encouraged). +- **Protocols folder**: Contains the protocols that describe the process from samples to measurement. + +#### Workflows +The **workflows** folder contains subfolders for each workflow, which may include anything from simple scripts to full programs or toolchains for simulations, processing, or analysis. These workflows should not be tied to specific input/output files, which are instead managed by the metadata in a run. +- **Workflow metadata file**: Describes the executables and computational environment needed for the workflow. + +#### Runs +For each computational run, a separate folder is created to store the resulting data. +- **Run metadata file**: Contains specific parameter values for the run, including the input data used. + +### Flexibility and Expansion +The ARC scaffold provides a well-defined space for organizing research data but does not require every aspect to be filled. Researchers and collaborators can use this structure as needed, leaving any irrelevant sections empty. Additional folders can be created for other research elements, such as paper drafts or notes, allowing for flexible expansion. + +For more detailed information, refer to the [ARC Scaffold Specification]. + +### Continuous Quality Control +ARC supports continuous quality control in the background, ensuring data integrity without interrupting work. The entire investigation can be compiled into a data publication, assigned a DOI, and referenced in journal publications. +