Merge pull request #3374 from AleoHQ/feat/network-retriever

[Draft] Network retriever module

Cargo.toml

@@ -42,7 +42,8 @@ members = [
   "errors",
   "leo/package",
   "tests/test-framework",
-  "utils/disassembler"
+  "utils/disassembler",
+  "utils/retriever"
 ]
 
 [workspace.dependencies.snarkvm]
@@ -84,6 +85,10 @@ version = "=1.10.0"
 path = "./compiler/span"
 version = "=1.10.0"
 
+[dependencies.retriever]
+path = "./utils/retriever"
+version = "1.10.0"
+
 [dependencies.backtrace]
 version = "0.3.68"
 
@@ -123,10 +128,6 @@ default-features = false
 [dependencies.rand_core]
 version = "0.6.4"
 
-[dependencies.reqwest]
-version = "0.11.22"
-features = [ "blocking", "json", "multipart" ]
-
 [dependencies.self_update]
 version = "0.39.0"
 features = [ "archive-zip" ]

compiler/ast/src/stub/mod.rs

@@ -21,7 +21,7 @@ pub use finalize_stub::*;
 pub mod function_stub;
 pub use function_stub::*;
 
-use crate::{ConstDeclaration, Mapping, ProgramId, Struct};
+use crate::{ConstDeclaration, Identifier, Mapping, NodeID, ProgramId, Struct};
 use leo_span::{Span, Symbol};
 use serde::{Deserialize, Serialize};
 use std::fmt;
@@ -45,6 +45,24 @@ pub struct Stub {
     pub span: Span,
 }
 
+impl Default for Stub {
+    /// Constructs an empty program stub
+    fn default() -> Self {
+        Self {
+            imports: Vec::new(),
+            stub_id: ProgramId {
+                name: Identifier::new(Symbol::intern(""), NodeID::default()),
+                network: Identifier::new(Symbol::intern(""), NodeID::default()),
+            },
+            consts: Vec::new(),
+            structs: Vec::new(),
+            mappings: Vec::new(),
+            functions: Vec::new(),
+            span: Span::default(),
+        }
+    }
+}
+
 impl fmt::Display for Stub {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         writeln!(f, "stub {} {{", self.stub_id)?;

compiler/compiler/Cargo.toml

@@ -41,6 +41,10 @@ version = "=1.10.0"
 [dependencies.sha2]
 version = "0.10"
 
+[dependencies.indexmap]
+version = "1.9"
+features = []
+
 [dev-dependencies.leo-test-framework]
 path = "../../tests/test-framework"
 
@@ -72,4 +76,4 @@ version = "3.8"
 
 [features]
 default = [ ]
-ci_skip = [ "leo-ast/ci_skip" ]
+ci_skip = [ "leo-ast/ci_skip" ]

compiler/compiler/src/compiler.rs

@@ -18,16 +18,17 @@
 //!
 //! The [`Compiler`] type compiles Leo programs into R1CS circuits.
 pub use leo_ast::{Ast, InputAst};
-use leo_ast::{NodeBuilder, Program};
+use leo_ast::{NodeBuilder, Program, Stub};
 use leo_errors::{emitter::Handler, CompilerError, Result};
 pub use leo_passes::SymbolTable;
 use leo_passes::*;
-use leo_span::{source_map::FileName, symbol::with_session_globals};
+use leo_span::{source_map::FileName, symbol::with_session_globals, Symbol};
 
 use sha2::{Digest, Sha256};
 use std::{fs, path::PathBuf};
 
 use crate::CompilerOptions;
+use indexmap::{IndexMap, IndexSet};
 
 /// The primary entry point of the Leo compiler.
 #[derive(Clone)]
@@ -54,6 +55,8 @@ pub struct Compiler<'a> {
     assigner: Assigner,
     /// The type table.
     type_table: TypeTable,
+    /// The stubs for imported programs. Produced by `Retriever` module.
+    import_stubs: IndexMap<Symbol, Stub>,
 }
 
 impl<'a> Compiler<'a> {
@@ -65,6 +68,7 @@ impl<'a> Compiler<'a> {
         main_file_path: PathBuf,
         output_directory: PathBuf,
         compiler_options: Option<CompilerOptions>,
+        import_stubs: IndexMap<Symbol, Stub>,
     ) -> Self {
         let node_builder = NodeBuilder::default();
         let assigner = Assigner::default();
@@ -80,6 +84,7 @@ impl<'a> Compiler<'a> {
             compiler_options: compiler_options.unwrap_or_default(),
             node_builder,
             assigner,
+            import_stubs,
             type_table,
         }
     }
@@ -324,6 +329,8 @@ impl<'a> Compiler<'a> {
     pub fn compile(&mut self) -> Result<(SymbolTable, String)> {
         // Parse the program.
         self.parse_program()?;
+        // Copy the dependencies specified in `program.json` into the AST.
+        self.add_import_stubs()?;
         // Run the intermediate compiler stages.
         let (symbol_table, struct_graph, call_graph) = self.compiler_stages()?;
         // Run code generation.
@@ -361,4 +368,41 @@ impl<'a> Compiler<'a> {
         }
         Ok(())
     }
+
+    /// Merges the dependencies defined in `program.json` with the dependencies imported in `.leo` file
+    fn add_import_stubs(&mut self) -> Result<()> {
+        // Create a list of both the explicit dependencies specified in the `.leo` file, as well as the implicit ones derived from those dependencies.
+        let (mut unexplored, mut explored): (IndexSet<Symbol>, IndexSet<Symbol>) =
+            (self.ast.ast.imports.keys().cloned().collect(), IndexSet::new());
+        while !unexplored.is_empty() {
+            let mut current_dependencies: IndexSet<Symbol> = IndexSet::new();
+            for program_name in unexplored.iter() {
+                if let Some(stub) = self.import_stubs.get(program_name) {
+                    // Add the program to the explored set
+                    explored.insert(*program_name);
+                    for dependency in stub.imports.iter() {
+                        // If dependency is already explored then don't need to re-explore it
+                        if explored.insert(dependency.name.name) {
+                            current_dependencies.insert(dependency.name.name);
+                        }
+                    }
+                } else {
+                    return Err(CompilerError::imported_program_not_found(
+                        self.program_name.clone(),
+                        *program_name,
+                        self.ast.ast.imports[program_name].1,
+                    )
+                    .into());
+                }
+            }
+
+            // Create next batch to explore
+            unexplored = current_dependencies;
+        }
+
+        // Combine the dependencies from `program.json` and `.leo` file while preserving the post-order
+        self.ast.ast.stubs =
+            self.import_stubs.clone().into_iter().filter(|(program_name, _)| explored.contains(program_name)).collect();
+        Ok(())
+    }
 }

compiler/compiler/tests/utilities/mod.rs

@@ -29,6 +29,7 @@ use leo_test_framework::{test::TestConfig, Test};
 
 use snarkvm::prelude::*;
 
+use indexmap::IndexMap;
 use leo_ast::ProgramVisitor;
 use snarkvm::{file::Manifest, package::Package};
 use std::{
@@ -142,7 +143,15 @@ pub fn new_compiler(
     let output_dir = PathBuf::from("/tmp/output/");
     fs::create_dir_all(output_dir.clone()).unwrap();
 
-    Compiler::new(String::from("test"), String::from("aleo"), handler, main_file_path, output_dir, compiler_options)
+    Compiler::new(
+        String::from("test"),
+        String::from("aleo"),
+        handler,
+        main_file_path,
+        output_dir,
+        compiler_options,
+        IndexMap::new(),
+    )
 }
 
 pub fn parse_program<'a>(

compiler/parser/src/parser/file.rs

@@ -412,7 +412,7 @@ impl ParserContext<'_> {
         };
 
         // Parse the function body. Allow empty blocks. `fn foo(a:u8);`
-        let (has_empty_block, block) = match &self.token.token {
+        let (_has_empty_block, block) = match &self.token.token {
             Token::LeftCurly => (false, self.parse_block()?),
             Token::Semicolon => {
                 let semicolon = self.expect(&Token::Semicolon)?;
@@ -425,11 +425,6 @@ impl ParserContext<'_> {
         let finalize = match self.eat(&Token::Finalize) {
             false => None,
             true => {
-                // Make sure has function body. Don't want `fn foo(); finalize foo { ... }` to be valid parsing.
-                if has_empty_block {
-                    return Err(ParserError::empty_function_cannot_have_finalize(self.token.span).into());
-                }
-
                 // Get starting span.
                 let start = self.prev_token.span;
 

compiler/passes/src/code_generation/visit_program.rs

@@ -16,7 +16,7 @@
 
 use crate::CodeGenerator;
 
-use leo_ast::{functions, Function, Mapping, Mode, Program, ProgramScope, Struct, Stub, Type, Variant};
+use leo_ast::{functions, Function, Mapping, Mode, Program, ProgramScope, Struct, Type, Variant};
 
 use indexmap::IndexMap;
 use itertools::Itertools;
@@ -28,22 +28,10 @@ impl<'a> CodeGenerator<'a> {
         // Accumulate instructions into a program string.
         let mut program_string = String::new();
 
-        if !input.imports.is_empty() {
-            // Visit each import statement and produce a Aleo import instruction.
-            program_string.push_str(
-                &input
-                    .imports
-                    .iter()
-                    .map(|(identifier, (imported_program, _))| self.visit_import(identifier, imported_program))
-                    .join("\n"),
-            );
-
-            // Newline separator.
-            program_string.push('\n');
-        }
-
-        // Import stub programs
-        program_string.push_str(&input.stubs.values().map(|stub| self.visit_stub(stub)).join("\n"));
+        // Print out the dependencies of the program. Already arranged in post order by Retriever module.
+        input.stubs.iter().for_each(|(program_name, _)| {
+            program_string.push_str(&format!("import {}.aleo;\n", program_name));
+        });
 
         // Retrieve the program scope.
         // Note that type checking guarantees that there is exactly one program scope.
@@ -112,15 +100,6 @@ impl<'a> CodeGenerator<'a> {
         program_string
     }
 
-    fn visit_stub(&mut self, input: &'a Stub) -> String {
-        format!("import {}.aleo;", input.stub_id.name)
-    }
-
-    fn visit_import(&mut self, import_name: &'a Symbol, _import_program: &'a Program) -> String {
-        // Generate string for import statement.
-        format!("import {import_name}.aleo;")
-    }
-
     fn visit_struct_or_record(&mut self, struct_: &'a Struct) -> String {
         if struct_.is_record { self.visit_record(struct_) } else { self.visit_struct(struct_) }
     }