From a0da0bf941e2d8b56f961e124cefa23d8b58fe95 Mon Sep 17 00:00:00 2001 From: Alexander Devitsky Date: Wed, 22 Sep 2021 13:34:11 +0300 Subject: [PATCH 1/7] Added cache groups --- helpers.js | 29 +++++++++++++++++++++++++---- sync.js | 16 ++++++++++++++-- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/helpers.js b/helpers.js index 96de063..695abe7 100644 --- a/helpers.js +++ b/helpers.js @@ -61,9 +61,27 @@ export const transformId = (id) => { } } +export const getCacheTag = (story) => { + const slugParts = story.full_slug.split('/') + let cacheTag = '' + + if (slugParts[0] === 'system') { + cacheTag = 'SB_SYS' + } else if (slugParts[0] === 'overrides' && slugParts[1].includes('brand-') && slugParts[2] === 'system') { + cacheTag = story.full_slug.includes('system/general') + ? `SB${slugParts[1].substring('brand-'.length)}` + : 'SB_SYS' + } else { + cacheTag = `SB${story.id}` + } + + return cacheTag +} + export const transformStory = ({ id, ...story } = {}) => { story.content = JSON.stringify(story.content) story.full_slug = story.full_slug.replace(/^\/|\/$/g, '') + story.cache_tag = getCacheTag(story) return { ...transformId(id), body: story @@ -131,12 +149,15 @@ export const log = (string) => { console.log('📖 : ' + string) // eslint-disable-line no-console } -export const cacheInvalidate = async () => { +export const cacheInvalidate = async (config, story = null) => { if (config.invalidate) { + const url = config.invalidate + if (story && story.cache_tag) { + const queryParams = new URLSearchParams(url) + queryParams.set('tag', story.cache_tag) + } log(`Invalidating cache... (${config.invalidate})`) - await rp({ - uri: config.invalidate - }) + await rp({ uri: url }) log('Invalidated cache ✅') } } diff --git a/sync.js b/sync.js index 0650c4b..89d5f79 100644 --- a/sync.js +++ b/sync.js @@ -1,5 +1,14 @@ import { storyblokClient } from './storyblok' -import { log, createIndex, deleteIndex, createBulkOperations, transformId, transformStory, cacheInvalidate } from './helpers' +import { + log, + createIndex, + deleteIndex, + createBulkOperations, + transformId, + transformStory, + cacheInvalidate, + getCacheTag +} from './helpers' function indexStories ({ db, stories = [] }) { const bulkOps = createBulkOperations(stories) @@ -28,6 +37,7 @@ async function syncStories ({ db, page = 1, perPage = 100, environment = null }) ...story, full_slug: fullSlug, real_path: fullSlug.substr(0, 1) === '/' ? fullSlug : `/${fullSlug}`, + cache_tag: getCacheTag(story), folder: fullSlug.lastIndexOf('/') !== -1 ? fullSlug.substring(0, fullSlug.lastIndexOf('/')) : null } }) @@ -55,6 +65,7 @@ const fullSync = async (db, config) => { const handleHook = async (db, config, params) => { const cv = Date.now() // bust cache const { story_id: id, action } = params + let invalidatedStory = null switch (action) { case 'published': @@ -82,6 +93,7 @@ const handleHook = async (db, config, params) => { } const publishedStory = transformStory(story) + invalidatedStory = publishedStory await db.index(publishedStory) log(`Published ${story.full_slug}`) @@ -99,7 +111,7 @@ const handleHook = async (db, config, params) => { default: break } - await cacheInvalidate(config.storyblok) + await cacheInvalidate(config.storyblok, invalidatedStory) } const seedDatabase = async (db, config) => { From 2e3a8571d9266713d4fd85a7709c5695b32e90a9 Mon Sep 17 00:00:00 2001 From: Alexander Devitsky Date: Thu, 30 Sep 2021 12:50:00 +0300 Subject: [PATCH 2/7] Updated cache tags --- helpers.js | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/helpers.js b/helpers.js index 695abe7..7c3ba99 100644 --- a/helpers.js +++ b/helpers.js @@ -62,20 +62,7 @@ export const transformId = (id) => { } export const getCacheTag = (story) => { - const slugParts = story.full_slug.split('/') - let cacheTag = '' - - if (slugParts[0] === 'system') { - cacheTag = 'SB_SYS' - } else if (slugParts[0] === 'overrides' && slugParts[1].includes('brand-') && slugParts[2] === 'system') { - cacheTag = story.full_slug.includes('system/general') - ? `SB${slugParts[1].substring('brand-'.length)}` - : 'SB_SYS' - } else { - cacheTag = `SB${story.id}` - } - - return cacheTag + return story.id ? `SB${story.id}` : '' } export const transformStory = ({ id, ...story } = {}) => { From d0a952054120c6b7117c6899236f64803e0300f7 Mon Sep 17 00:00:00 2001 From: Georgiy Slobodenyuk Date: Sat, 4 Sep 2021 23:32:20 +0000 Subject: [PATCH 3/7] fix: Improve reliability and stability on startup and sync --- index.js | 2 +- sync.js | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/index.js b/index.js index e439a9e..06addc5 100644 --- a/index.js +++ b/index.js @@ -15,7 +15,7 @@ module.exports = ({ config }) => { setConfig(config) initStoryblokClient(config) - seedDatabase(db, config) + api.get('/story/', async (req, res) => { const story = await getStory(db, 'home') diff --git a/sync.js b/sync.js index 89d5f79..d3a1e55 100644 --- a/sync.js +++ b/sync.js @@ -57,9 +57,16 @@ async function syncStories ({ db, page = 1, perPage = 100, environment = null }) const fullSync = async (db, config) => { log('Syncing published stories!') - await db.indices.delete(deleteIndex()) - await db.indices.create(createIndex()) - await syncStories({ db, perPage: config.storyblok.perPage, environment: config.storyblok.environment }) + // This will call the storyblok API, load up the stories, and then return a promise to index them + // Tested with ngrok + try { + const indexStories = syncStories({ db, perPage: config.storyblok.perPage, environment: config.storyblok.environment }) + await db.indices.delete(deleteIndex()) + await db.indices.create(createIndex()) + await indexStories + } catch(e) { + console.error(e) + } } const handleHook = async (db, config, params) => { From 007ee9a01982490f4bb386171e8b73fa9e3113c9 Mon Sep 17 00:00:00 2001 From: Alexander Devitsky Date: Tue, 5 Oct 2021 14:35:47 +0300 Subject: [PATCH 4/7] fix: Fixed missing id in story --- helpers.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helpers.js b/helpers.js index 7c3ba99..46cd5d0 100644 --- a/helpers.js +++ b/helpers.js @@ -65,12 +65,12 @@ export const getCacheTag = (story) => { return story.id ? `SB${story.id}` : '' } -export const transformStory = ({ id, ...story } = {}) => { +export const transformStory = (story = {}) => { story.content = JSON.stringify(story.content) story.full_slug = story.full_slug.replace(/^\/|\/$/g, '') story.cache_tag = getCacheTag(story) return { - ...transformId(id), + ...transformId(story.id), body: story } } From e99a064b79f1f3e09802a961798c5f409b2bdb87 Mon Sep 17 00:00:00 2001 From: Alexander Devitsky Date: Tue, 5 Oct 2021 19:13:48 +0300 Subject: [PATCH 5/7] Added invalidate flag --- helpers.js | 11 +++++++---- index.js | 5 +++-- sync.js | 6 ++++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/helpers.js b/helpers.js index 46cd5d0..561f5ad 100644 --- a/helpers.js +++ b/helpers.js @@ -66,12 +66,15 @@ export const getCacheTag = (story) => { } export const transformStory = (story = {}) => { - story.content = JSON.stringify(story.content) - story.full_slug = story.full_slug.replace(/^\/|\/$/g, '') - story.cache_tag = getCacheTag(story) + const storyParams = { + content: JSON.stringify(story.content), + full_slug: story.full_slug.replace(/^\/|\/$/g, ''), + cache_tag: getCacheTag(story) + } + const transformedStory = Object.assign({}, story, storyParams) return { ...transformId(story.id), - body: story + body: transformedStory } } diff --git a/index.js b/index.js index 06addc5..e819c61 100644 --- a/index.js +++ b/index.js @@ -15,7 +15,7 @@ module.exports = ({ config }) => { setConfig(config) initStoryblokClient(config) - + api.get('/story/', async (req, res) => { const story = await getStory(db, 'home') @@ -44,7 +44,8 @@ module.exports = ({ config }) => { api.post('/hook', protectRoute(config), async (req, res) => { try { - await handleHook(db, config, req.body) + const { invalidate = false } = req.query + await handleHook(db, config, req.body, invalidate) apiStatus(res) } catch (error) { apiStatus(res, { diff --git a/sync.js b/sync.js index d3a1e55..599055d 100644 --- a/sync.js +++ b/sync.js @@ -69,7 +69,7 @@ const fullSync = async (db, config) => { } } -const handleHook = async (db, config, params) => { +const handleHook = async (db, config, params, invalidate) => { const cv = Date.now() // bust cache const { story_id: id, action } = params let invalidatedStory = null @@ -118,7 +118,9 @@ const handleHook = async (db, config, params) => { default: break } - await cacheInvalidate(config.storyblok, invalidatedStory) + if (invalidate) { + await cacheInvalidate(config.storyblok, invalidatedStory) + } } const seedDatabase = async (db, config) => { From 7f002a694407846de07866ea77f3350d7d38a4cc Mon Sep 17 00:00:00 2001 From: Georgiy Slobodenyuk Date: Wed, 6 Oct 2021 23:11:15 +0000 Subject: [PATCH 6/7] fix(AMUSE-3600): Properly invalidate the incoming stories cache-tag --- helpers.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/helpers.js b/helpers.js index 561f5ad..6127ae7 100644 --- a/helpers.js +++ b/helpers.js @@ -141,12 +141,11 @@ export const log = (string) => { export const cacheInvalidate = async (config, story = null) => { if (config.invalidate) { - const url = config.invalidate + let url = config.invalidate if (story && story.cache_tag) { - const queryParams = new URLSearchParams(url) - queryParams.set('tag', story.cache_tag) + url += story.cache_tag; } - log(`Invalidating cache... (${config.invalidate})`) + log(`Invalidating cache... (${url})`) await rp({ uri: url }) log('Invalidated cache ✅') } From dbe5f4a07648fd5cf874f13477ea2596350e7e95 Mon Sep 17 00:00:00 2001 From: Georgiy Slobodenyuk Date: Thu, 7 Oct 2021 16:28:00 +0000 Subject: [PATCH 7/7] fix(AMUSE-3600): Fix cache-tag reference --- helpers.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helpers.js b/helpers.js index 6127ae7..9c8b99c 100644 --- a/helpers.js +++ b/helpers.js @@ -142,8 +142,8 @@ export const log = (string) => { export const cacheInvalidate = async (config, story = null) => { if (config.invalidate) { let url = config.invalidate - if (story && story.cache_tag) { - url += story.cache_tag; + if (story && story.body.cache_tag) { + url += story.body.cache_tag; } log(`Invalidating cache... (${url})`) await rp({ uri: url })