diff --git a/base.lua b/base.lua index 18a46f1..ecfc7d6 100644 --- a/base.lua +++ b/base.lua @@ -21,7 +21,7 @@ if not home_store then file:write(home_store) file:close() end -content.home = home_store +contentdb.home = home_store if torchbear.settings.theme then theme_loader.load_themes("themes/", torchbear.settings.theme) @@ -51,4 +51,4 @@ outgoing_response_event:setActionPriority(response_process_action, 100) log.info("[loaded] LightTouch") -events["lighttouch_loaded"]:trigger() \ No newline at end of file +events["lighttouch_loaded"]:trigger() diff --git a/content/add_document_to_index.lua b/content/add_document_to_index.lua deleted file mode 100644 index fff895f..0000000 --- a/content/add_document_to_index.lua +++ /dev/null @@ -1,13 +0,0 @@ -function content.add_document_to_index (doc_id, store_id, file_content, model) - if not model then - log.warn("Document " .. doc_id .. " in " .. store_id .. " does not have a model") - return - end - - local doc = tan.new_document() - doc:add_text(content.fields.uuid, doc_id) - doc:add_text(content.fields.store, store_id) - doc:add_text(content.fields.model, model) - doc:add_text(content.fields.content, file_content) - content.index_writer:add_document(doc) -end \ No newline at end of file diff --git a/content/base.lua b/content/base.lua deleted file mode 100644 index 610cf7f..0000000 --- a/content/base.lua +++ /dev/null @@ -1,28 +0,0 @@ -content = { - stores = { - home = "content/home/" - } -} - --- Methods for the model class -local model_metatable = {} -model_metatable.__index = model_metatable -function model_metatable:validate (object) - for name, validator in pairs(self.fields) do - local result, err = validator(object[name]) - if not result then - return result, name .. ": " .. err - end - end - return true -end - --- Add all directories in content to the stores table -for entry in fs.entries("content/") do - if fs.metadata("content/" .. entry).type == "directory" then - content.stores[entry] = "content/" .. entry .. "/" - end -end - - - fs.create_dir("content/home", true) diff --git a/content/documents.lua b/content/documents.lua deleted file mode 100644 index 7b1d96b..0000000 --- a/content/documents.lua +++ /dev/null @@ -1,33 +0,0 @@ -function content.documents (store_id) - if store_id then - local dir = content.stores[store_id] - - local f = fs.entries(dir) - - return function () - local entry = f() - if entry then - return entry, store_id - end - end - else - - local docs_co = coroutine.create(function () - for store_id, dir in pairs(content.stores) do - if fs.exists(dir) then - for entry in fs.entries(dir) do - coroutine.yield(entry, store_id) - end - end - end - end) - - return function () - local cont, uuid, store_id = coroutine.resume(docs_co) - if cont then - return uuid, store_id - end - end - - end -end \ No newline at end of file diff --git a/content/get_document_path.lua b/content/get_document_path.lua deleted file mode 100644 index 972ef9b..0000000 --- a/content/get_document_path.lua +++ /dev/null @@ -1,9 +0,0 @@ -function content.get_document_path (doc_uuid, store_id) - if store_id then - return (content.stores[store_id] or "content/" .. store_id) .. doc_uuid - else - for store_id, dir in content.stores_iter() do - - end - end -end \ No newline at end of file diff --git a/content/get_validator.lua b/content/get_validator.lua deleted file mode 100644 index 1b72fab..0000000 --- a/content/get_validator.lua +++ /dev/null @@ -1,32 +0,0 @@ -function content.get_validator (name) - local model_def = models[name] - if not model_def then - return nil, "Model " .. name .. " not found" - end - - for name, field_def in pairs(model_def.fields) do - local validator = valua:new() - - if field_def.optional then - validator.optional(true) - end - - if field_def.type == "number" then - validator.number() - elseif field_def.type == "integer" then - validator.integer() - elseif field_def.type == "string" then - validator.string() - else - return nil, "missing type in field " .. name - end - - fields[name] = validator - end - - local model = setmetatable({ - fields = fields - }, model_metatable) - - return model -end \ No newline at end of file diff --git a/content/mod.lua b/content/mod.lua deleted file mode 100644 index a1252cc..0000000 --- a/content/mod.lua +++ /dev/null @@ -1,11 +0,0 @@ -require "content.base" -require "content.split_header" -require "content.get_validator" -require "content.validate_document" -require "content.read_document" -require "content.documents" -require "content.walk_documents" -require "content.write_file" -local valua = require "third-party.valua" - -return content diff --git a/content/query.lua b/content/query.lua deleted file mode 100644 index 0b18dbd..0000000 --- a/content/query.lua +++ /dev/null @@ -1,24 +0,0 @@ -function content.query (query_str) - log.trace("Start tantivy search") - - local fields = {} - for _, v in pairs(content.fields) do - table.insert(fields, v) - end - - local parser = tan.query_parser_for_index(content.index, fields) - local coll = tan.top_collector_with_limit(500) - local result = content.index:search(parser, query_str, coll) - - return result - --[[ - for i = 1, #result do - local doc = result[i] - table.insert(uuids, { - file = doc:get_first(uuid_field), - profile = "unknown_profile" - }) - end - log.trace("End tantivy search") - --]] -end \ No newline at end of file diff --git a/content/read_document.lua b/content/read_document.lua deleted file mode 100644 index ceb4e75..0000000 --- a/content/read_document.lua +++ /dev/null @@ -1,8 +0,0 @@ -function content.read_document (in_uuid) - log.trace("Running: " .. debug.getinfo(1, 'S').source) - return content.walk_documents(nil, function (file_uuid, header, body, profile) - if file_uuid == in_uuid then - return header, body, profile - end - end) -end diff --git a/content/read_log.lua b/content/read_log.lua deleted file mode 100644 index 88da6d3..0000000 --- a/content/read_log.lua +++ /dev/null @@ -1,15 +0,0 @@ -function content.read_log (log_uuid) - -- It's getting just the request logs, it should read all logs but this - -- function is not completely thought trough yet - local dir = "log/incoming-request/" - - for _, file_uuid in ipairs(fs.get_all_files_in(dir)) do - local path = dir .. file_uuid - local file_content = fs.read_file(path) - if not file_content then - error("could not open " .. path) - end - - return scl.to_table(file_content) - end -end diff --git a/content/setup_index.lua b/content/setup_index.lua deleted file mode 100644 index 6ab3343..0000000 --- a/content/setup_index.lua +++ /dev/null @@ -1,39 +0,0 @@ -function content.setup_index (path) - path = path or "./tantivy-index" - - --TODO: implement fs.remove_dir(path, true) - --os.execute("rm -r " .. path) - --fs.create_dir(path, true) - - content.index = tan.index_in_ram(content.schema) - - content.index_writer = content.index:writer(50000000) - - -- Walk through all documents and add them to the index - for store_id, dir in pairs(content.stores) do - if fs.exists(dir) then - for doc_id in fs.entries(dir) do - - -- Would use content.read_document but it doesn't return the file contents - local path = content.stores[store_id] .. doc_id - - local file_content = fs.read_file(path) - if not file_content then - error("could not open " .. path) - end - - local doc_fields = content.split_header(file_content) - - content.add_document_to_index( - doc_id, - store_id, - file_content, - doc_fields.model - ) - end - end - end - - content.index_writer:commit() - content.index:load_searchers() -end \ No newline at end of file diff --git a/content/setup_schema.lua b/content/setup_schema.lua deleted file mode 100644 index 6bb2bc0..0000000 --- a/content/setup_schema.lua +++ /dev/null @@ -1,17 +0,0 @@ -function content.setup_schema () - local builder = tan.new_schema_builder() - - builder:add_text_field("uuid", {tan.STRING, tan.STORED}) - builder:add_text_field("store", {tan.STRING, tan.STORED}) - builder:add_text_field("model", {tan.STRING, tan.STORED}) - builder:add_text_field("content", {tan.TEXT}) - - content.schema = builder:build() - - content.fields = { - uuid = content.schema:get_field("uuid"), - store = content.schema:get_field("store"), - model = content.schema:get_field("model"), - content = content.schema:get_field("content"), - } -end diff --git a/content/split_header.lua b/content/split_header.lua deleted file mode 100644 index 9c753b3..0000000 --- a/content/split_header.lua +++ /dev/null @@ -1,9 +0,0 @@ -function content.split_header (document_text) - log.trace("Running: " .. debug.getinfo(1, 'S').source) - local scl_text, body = document_text:match("---\n(.*)\n+...\n(.*)") - if scl_text == nil then - scl_text = "" - end - local header = scl.to_table(scl_text) - return header, body -end diff --git a/content/validate_document.lua b/content/validate_document.lua deleted file mode 100644 index 47c2b0a..0000000 --- a/content/validate_document.lua +++ /dev/null @@ -1,12 +0,0 @@ -function content.validate_document (header) - if not header.model then - return false, "document does not define a model" - end - - local model, err = content.get_validator(header.model) - if not model then - return false, err - end - - return model:validate(header) -end \ No newline at end of file diff --git a/content/walk_documents.lua b/content/walk_documents.lua deleted file mode 100644 index 83278cb..0000000 --- a/content/walk_documents.lua +++ /dev/null @@ -1,19 +0,0 @@ -function content.walk_documents (_store_id, fn) - log.trace("Running: " .. debug.getinfo(1, 'S').source) - for doc_id, store_id in content.documents(_store_id) do - - local path = content.stores[store_id] .. doc_id - local file_content = fs.read_file(path) - if not file_content then - log.error("could not open " .. path) - end - - local header, body = content.split_header(file_content) - - -- If the fn applied on this file returns values, stop walking - local results = { fn(doc_id, header, body, store_id) } - if results[1] then - return table.unpack(results) - end - end -end diff --git a/content/write_file.lua b/content/write_file.lua deleted file mode 100644 index b0d7b32..0000000 --- a/content/write_file.lua +++ /dev/null @@ -1,30 +0,0 @@ -function content.write_file (store_id, file_uuid, fields, body_param) - log.trace("Running: " .. debug.getinfo(1, 'S').source) - - local dir = content.stores[store_id] - if not dir then - dir = "content/" .. store_id .. "/" - fs.create_dir(dir, true) - content.stores[store_id] = dir - end - local path = dir .. file_uuid - - if not fields.creation_time then - fields.creation_time = tostring(time.now()) - end - - local body = "---\n" .. scl.from_table(fields) .. "...\n" .. (body_param or "") - local file = io.open(path, "w") - if not file then - log.error("Could not open file", path) - end - file:write(body) - file:close() - - events["document_created"]:trigger({ - store_id = store_id, - file_uuid = file_uuid, - fields = header, - body = body - }) -end diff --git a/fs.lua b/fs.lua deleted file mode 100644 index ba777fa..0000000 --- a/fs.lua +++ /dev/null @@ -1,109 +0,0 @@ -function fs.to_dir_name (path) - if path.sub(-1, -1) ~= "/" then - path = path .. "/" - end - return path -end - -function fs.read_file(path) - local file = io.open(path, "r") - - if not file then - return nil - end - - local file_content = file:read("*all") - file:close() - return file_content -end - --- Basically io.lines except it doesn't fails if the file doesn't exist -function fs.read_lines(path, ...) - local file = io.open(path, "r") - - -- Returning an empty function makes the for loop do nothing - if not file then - return function () end - end - - local _f, s, var = file:lines(...) - - -- Returns the parameters as they are but if the first parameter is nil closes the file - -- (lua loops stop after the first result of the iterator function is nil) - local function bridge (var1, ...) - if var1 == nil then - file:close() - end - return var1, ... - end - - -- A wrapper that closes the file when the iterator ends - local function f (s, var) - return bridge(_f(s, var)) - end - - return f, s, var -end - -function fs.get_all_files_in(directory) - - if directory.sub(-1, -1) ~= "/" then - directory = directory .. "/" - end - - local filenames = {} - - for filename in fs.entries(directory) do - if fs.metadata(directory .. filename).type == "file" then - table.insert(filenames, filename) - end - end - - return filenames -end - -function fs.directory_list(directory) - - if directory.sub(-1, -1) ~= "/" then - directory = directory .. "/" - end - - local filenames = {} - - for filename in fs.entries(directory) do - if fs.metadata(directory .. filename).type == "directory" then - table.insert(filenames, filename) - end - end - - return filenames -end - -function fs.copy(src_path, dest_path) - local src_file = assert(io.open(src_path, "r")) - local dest_file = assert(io.open(dest_path, "w+")) - - dest_file:write(src_file:read("*all")) - - src_file:close() - dest_file:close() -end - -function fs.append_to_start(file_path, to_append) - local file = assert(io.open(file_path, "r")) - local file_content = file:read("*all") - file:close() - - file = assert(io.open(file_path, "w")) - file:write(to_append) - file:write("\n") - file:write(file_content) - - file:close() -end - -function fs.exists (path) - return fs.metadata(path) ~= nil -end - -return fs \ No newline at end of file diff --git a/import.scl b/import.scl index e8bb511..5b64bd6 100644 --- a/import.scl +++ b/import.scl @@ -1,25 +1,30 @@ imports = { luvent = "third-party/", valua = "third-party/", - torchbear-libs = "third-party/", - # torchbear-libs-dev = "third-party/", + torchbear-libs = "third-party/", + contentdb-lua = "contentdb", + torchbear-libs-dev = "third-party/", } torchbear-libs = { - url = "https://github.com/foundpatterns/torchbear-libs", - repo = true, + url = "https://github.com/foundpatterns/torchbear-libs", + repo = true, } torchbear-libs-dev = { - url = "https://github.com/foundpatterns/torchbear-libs-dev", - repo = true, + url = "https://github.com/foundpatterns/torchbear-libs-dev", + repo = true, } luvent = { - url = "https://github.com/ejmr/Luvent", - export = "src" + url = "https://github.com/ejmr/Luvent", + export = "src" } valua = { - url = "https://github.com/sailorproject/valua", - pluck = "valua.lua", + url = "https://github.com/sailorproject/valua", + pluck = "valua.lua", +} + +contentdb-lua = { + url = "https://github.com/foundpatterns/contentdb.lua", } diff --git a/mod.lua b/mod.lua index a8d34de..0c5786b 100644 --- a/mod.lua +++ b/mod.lua @@ -2,12 +2,12 @@ _G.log = require "third-party.log" _G.inspect = require "third-party.inspect" _G.luvent = require "third-party.Luvent" require "third-party.base64" -_G.fs = require "fs" +_G.fs = require "third-party.fs" require "table_ext" require "string_ext" require "underscore_alias" -_G.content = require "content.mod" +_G.contentdb = require "contentdb.mod" _G.keys = require "keys.mod" require "loaders.package" theme_loader = require "loaders.themes.base" -class_loader = require "loaders.classes" \ No newline at end of file +class_loader = require "loaders.classes"