From 9e2ba6ffe1493305513811d92093bcf515badc5d Mon Sep 17 00:00:00 2001 From: ZnPdCo Date: Wed, 1 Jan 2025 10:38:10 +0800 Subject: [PATCH 1/4] =?UTF-8?q?fix:=20=E6=B7=BB=E5=8A=A0standard=E6=90=9C?= =?UTF-8?q?=E7=B4=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/main.js | 15 +++++++++------ build.sh | 36 +++++++++++++++++++++++++++++++----- webhook/index.js | 5 +++-- webhook/test.js | 10 +++++++++- 4 files changed, 52 insertions(+), 14 deletions(-) diff --git a/api/main.js b/api/main.js index 0fe92c7..48ed583 100644 --- a/api/main.js +++ b/api/main.js @@ -24,11 +24,6 @@ app.get('/status', function(req, res) { }); app.get('/', function(req, res) { - // if (!req.headers.referer || req.headers.referer.indexOf('oi-wiki.org') < 0) { - // res.send([]); - //return; - //} - // console.log(req.headers); if (!req.query.s) { res.send([]); return; @@ -37,7 +32,6 @@ app.get('/', function(req, res) { console.log(keyword); client.search({ index: "oiwiki", - type: "article", from: 0, size: 12, body: { @@ -79,6 +73,15 @@ app.get('/', function(req, res) { boost: 3 } } + }, + { + match: { + standard_content: { + query: keyword, + minimum_should_match: "75%", + boost: 2 + } + } } ], tie_breaker: 0.3 diff --git a/build.sh b/build.sh index 4dd045b..055e184 100644 --- a/build.sh +++ b/build.sh @@ -4,21 +4,47 @@ curl -H'Content-Type: application/json' -XPUT "http://localhost:9200/oiwiki" -d' "settings": { "analysis": { "analyzer": { - "default": { + "pinyin_analyzer": { "tokenizer": "ik_max_word", - "filter": "custom_pinyin" + "filter": "pinyin_filter" }, - "default_search": { + "pinyin_search_analyzer": { "tokenizer": "ik_max_word" } }, "filter": { - "custom_pinyin": { + "pinyin_filter": { "type": "pinyin", "keep_original": true, - "limit_first_letter_length": 16 + "limit_first_letter_length": 16, + "keep_joined_full_pinyin": true } } } + }, + "mappings": { + "properties": { + "content": { + "type": "text", + "analyzer": "pinyin_analyzer", + "search_analyzer": "pinyin_search_analyzer" + }, + "h2": { + "type": "text", + "analyzer": "pinyin_analyzer", + "search_analyzer": "pinyin_search_analyzer" + }, + "title": { + "type": "text", + "analyzer": "pinyin_analyzer", + "search_analyzer": "pinyin_search_analyzer" + }, + "url": { + "type": "text" + }, + "standard_content": { + "type": "text" + } + } } }' \ No newline at end of file diff --git a/webhook/index.js b/webhook/index.js index 8244e56..c0facef 100644 --- a/webhook/index.js +++ b/webhook/index.js @@ -90,17 +90,18 @@ function updateContent(modified, removed) { const data = YAML.parse(file.replaceAll('!!python/name:', '')); let ops = []; modified.forEach((filename) => { - ops.push({ index: { _index: 'oiwiki', _type: 'article', _id: filename } }); + ops.push({ index: { _index: 'oiwiki', _id: filename } }); let [title, article, h2] = getContent(filename, data); ops.push({ title: title, content: article, url: '/' + filename.replace('/index.md', '/').replace('.md', '/'), h2: h2, + standard_content: article, }); }); removed.forEach((filename) => { - ops.push({ delete: { _index: 'oiwiki', _type: 'article', _id: filename } }); + ops.push({ delete: { _index: 'oiwiki', _id: filename } }); }); client.bulk({ body: ops, refresh: 'true' }, function (err, res) { if (err) { diff --git a/webhook/test.js b/webhook/test.js index feab134..ff466ce 100644 --- a/webhook/test.js +++ b/webhook/test.js @@ -6,7 +6,6 @@ var client = new elasticsearch.Client({ let keyword = "线段"; const response = client.search({ index: "oiwiki", - type: "article", from: 0, size: 10, body: { @@ -48,6 +47,15 @@ const response = client.search({ boost: 3 } } + }, + { + match: { + standard_content: { + query: keyword, + minimum_should_match: "75%", + boost: 2 + } + } } ], tie_breaker: 0.3 From d92821a4179111213425880a5deb07b36d4e3a1c Mon Sep 17 00:00:00 2001 From: ZnPdCo Date: Mon, 17 Feb 2025 22:38:54 +0800 Subject: [PATCH 2/4] =?UTF-8?q?feat:=20=E5=AF=B9=E5=8A=A0=E7=B2=97?= =?UTF-8?q?=E9=83=A8=E5=88=86=E5=A2=9E=E5=8A=A0=E6=9D=83=E9=87=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/main.js | 9 +++++++++ build.sh | 5 +++++ webhook/index.js | 24 ++++++++++++++++-------- webhook/test.js | 9 +++++++++ 4 files changed, 39 insertions(+), 8 deletions(-) diff --git a/api/main.js b/api/main.js index 48ed583..357ca95 100644 --- a/api/main.js +++ b/api/main.js @@ -74,6 +74,15 @@ app.get('/', function(req, res) { } } }, + { + match: { + bold: { + query: keyword, + minimum_should_match: "75%", + boost: 4 + } + } + }, { match: { standard_content: { diff --git a/build.sh b/build.sh index 055e184..9dabde2 100644 --- a/build.sh +++ b/build.sh @@ -42,6 +42,11 @@ curl -H'Content-Type: application/json' -XPUT "http://localhost:9200/oiwiki" -d' "url": { "type": "text" }, + "bold": { + "type": "text", + "analyzer": "pinyin_analyzer", + "search_analyzer": "pinyin_search_analyzer" + }, "standard_content": { "type": "text" } diff --git a/webhook/index.js b/webhook/index.js index c0facef..59542cd 100644 --- a/webhook/index.js +++ b/webhook/index.js @@ -52,9 +52,9 @@ function getContent(filename, data) { return ['', '', '']; } - const h1reg = /^# .+$/gm, h2reg = /^## .+$/gm, authorreg = /author:[^\n]*/gm; + const h1reg = /^# .+$/gm, h2reg = /^## .+$/gm, authorreg = /author:[^\n]*/gm, boldreg = /\*\*(.*?)\*\*|__(.*?)__/g; const lines = file.split('\n').filter(e => !e.match(authorreg)); - let others = lines.filter((e) => !e.match(h1reg) && !e.match(h2reg)); + let content = lines.filter((e) => !e.match(h1reg) && !e.match(h2reg)); let title = lines[0] && lines[0].match(h1reg) ? lines[0].replace('# ', '') : ''; traversalArticle(data['nav'], (key, value) => { @@ -62,21 +62,28 @@ function getContent(filename, data) { }); const h2 = lines.filter(e => e.match(h2reg)).map(e => e.replace(/^## /, '')); - others = others.map(e => e.replace(/^##+ /, '')); + let bold = ""; + let match; + while ((match = boldreg.exec(file)) !== null) { + // match[1] is the content in **bold**, match[2] is the content in __bold__ + bold += match[1] || match[2]; + } + + content = content.map(e => e.replace(/^##+ /, '')); - remark.process(others.join('\n'), (err, file) => { + remark.process(content.join('\n'), (err, file) => { if (err) { console.error('Remark processing error:', err); return; } - others = String(file) + content = String(file) .replace('"', "") .replace("\\n\\n", "\\n"); }); - others.replace() + content.replace() - return [title, others, h2.join('\n')]; + return [title, content, h2.join('\n'), bold]; } /** @@ -91,12 +98,13 @@ function updateContent(modified, removed) { let ops = []; modified.forEach((filename) => { ops.push({ index: { _index: 'oiwiki', _id: filename } }); - let [title, article, h2] = getContent(filename, data); + let [title, article, h2, bold] = getContent(filename, data); ops.push({ title: title, content: article, url: '/' + filename.replace('/index.md', '/').replace('.md', '/'), h2: h2, + bold: bold, standard_content: article, }); }); diff --git a/webhook/test.js b/webhook/test.js index ff466ce..4c11b23 100644 --- a/webhook/test.js +++ b/webhook/test.js @@ -48,6 +48,15 @@ const response = client.search({ } } }, + { + match: { + bold: { + query: keyword, + minimum_should_match: "75%", + boost: 3 + } + } + }, { match: { standard_content: { From 237a88f8d9a54b909dc5f0f1a48a60982c45dadd Mon Sep 17 00:00:00 2001 From: ZnPdCo Date: Wed, 24 Sep 2025 13:59:04 +0800 Subject: [PATCH 3/4] upd --- README.md | 1 + api/main.js | 8 +- build.sh | 55 ------------- update.sh | 9 --- webhook/index.js | 199 +++++++++++++++++++++++++++++++---------------- 5 files changed, 135 insertions(+), 137 deletions(-) delete mode 100644 build.sh delete mode 100644 update.sh diff --git a/README.md b/README.md index e698d77..d65d879 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ npm install git clone https://github.com/OI-wiki/OI-wiki.git /home/ubuntu/OI-wiki cd /home/ubuntu/OI-wiki git remote add gh https://github.com/OI-wiki/OI-wiki.git +git clone https://github.com/OI-wiki/OI-wiki.git /var/www/OI-wiki -b gh-pages ``` 初始化并启动 webhook： diff --git a/api/main.js b/api/main.js index 357ca95..a743fa8 100644 --- a/api/main.js +++ b/api/main.js @@ -43,7 +43,7 @@ app.get('/', function(req, res) { title: { query: keyword, minimum_should_match: "75%", - boost: 4 + boost: 3 } } }, @@ -52,7 +52,7 @@ app.get('/', function(req, res) { h2: { query: keyword, minimum_should_match: "75%", - boost: 3 + boost: 2 } } }, @@ -79,7 +79,7 @@ app.get('/', function(req, res) { bold: { query: keyword, minimum_should_match: "75%", - boost: 4 + boost: 2 } } }, @@ -88,7 +88,7 @@ app.get('/', function(req, res) { standard_content: { query: keyword, minimum_should_match: "75%", - boost: 2 + boost: 4 } } } diff --git a/build.sh b/build.sh deleted file mode 100644 index 9dabde2..0000000 --- a/build.sh +++ /dev/null @@ -1,55 +0,0 @@ -curl -X DELETE "http://localhost:9200/oiwiki" -curl -H'Content-Type: application/json' -XPUT "http://localhost:9200/oiwiki" -d' -{ - "settings": { - "analysis": { - "analyzer": { - "pinyin_analyzer": { - "tokenizer": "ik_max_word", - "filter": "pinyin_filter" - }, - "pinyin_search_analyzer": { - "tokenizer": "ik_max_word" - } - }, - "filter": { - "pinyin_filter": { - "type": "pinyin", - "keep_original": true, - "limit_first_letter_length": 16, - "keep_joined_full_pinyin": true - } - } - } - }, - "mappings": { - "properties": { - "content": { - "type": "text", - "analyzer": "pinyin_analyzer", - "search_analyzer": "pinyin_search_analyzer" - }, - "h2": { - "type": "text", - "analyzer": "pinyin_analyzer", - "search_analyzer": "pinyin_search_analyzer" - }, - "title": { - "type": "text", - "analyzer": "pinyin_analyzer", - "search_analyzer": "pinyin_search_analyzer" - }, - "url": { - "type": "text" - }, - "bold": { - "type": "text", - "analyzer": "pinyin_analyzer", - "search_analyzer": "pinyin_search_analyzer" - }, - "standard_content": { - "type": "text" - } - } - } -}' \ No newline at end of file diff --git a/update.sh b/update.sh deleted file mode 100644 index ccf91cf..0000000 --- a/update.sh +++ /dev/null @@ -1,9 +0,0 @@ -cd /var/www/OI-wiki -# sleep 50s -git fetch origin gh-pages -git reset origin/gh-pages --hard -echo $USER - -cd /home/ubuntu/OI-wiki -git fetch gh master -git reset gh/master --hard \ No newline at end of file diff --git a/webhook/index.js b/webhook/index.js index 59542cd..9bf42fe 100644 --- a/webhook/index.js +++ b/webhook/index.js @@ -1,24 +1,32 @@ -var http = require('http'); -var YAML = require('yaml'); -var createHandler = require('github-webhook-handler'); -var handler = createHandler({ path: '/MY_SECRET_PATH', secret: 'MY_SECRET_KEY' }); +const http = require('http'); +const YAML = require('yaml'); +const createHandler = require('github-webhook-handler'); +const simpleGit = require('simple-git'); const elasticsearch = require('elasticsearch'); +const fs = require('fs'); +const path = require('path'); +const remarkLib = require('remark'); +const math = require('remark-math'); +const strip = require('strip-markdown-math'); + + +// const REPO_DIR = '/home/ubuntu/OI-wiki'; +const REPO_DIR = '/OI-wiki'; +const WEB_DIR = '/var/www/OI-wiki'; +const GITHUB_PATH = process.env.GITHUB_PATH || '/events'; +const GITHUB_SECRET = process.env.GITHUB_SECRET || 'asdlkfijawefojaewiofj'; + +const gitRepo = simpleGit({ baseDir: REPO_DIR }); +const gitWeb = simpleGit({ baseDir: WEB_DIR }); + var client = new elasticsearch.Client({ host: 'localhost:9200', }); - -let remark = require('remark'); -let strip = require('strip-markdown-math'); -const math = require("remark-math"); -remark = remark() +const remark = remarkLib() .use(math) .use(strip); -const fs = require('fs'); -const { exec } = require('child_process'); - - /** * Traversal all articles. * @@ -46,10 +54,10 @@ function traversalArticle(data, callback) { function getContent(filename, data) { let file; try { - file = String(fs.readFileSync(`/home/ubuntu/OI-wiki/docs/` + filename)); + file = String(fs.readFileSync(`/OI-wiki/docs/` + filename)); } catch (e) { console.error(`Error reading file ${filename}:`, e); - return ['', '', '']; + return ['', '', '', '']; } const h1reg = /^# .+$/gm, h2reg = /^## .+$/gm, authorreg = /author:[^\n]*/gm, boldreg = /\*\*(.*?)\*\*|__(.*?)__/g; @@ -62,11 +70,11 @@ function getContent(filename, data) { }); const h2 = lines.filter(e => e.match(h2reg)).map(e => e.replace(/^## /, '')); - let bold = ""; + let bold = []; let match; while ((match = boldreg.exec(file)) !== null) { // match[1] is the content in **bold**, match[2] is the content in __bold__ - bold += match[1] || match[2]; + bold.push(match[1] || match[2]); } content = content.map(e => e.replace(/^##+ /, '')); @@ -83,7 +91,7 @@ function getContent(filename, data) { content.replace() - return [title, content, h2.join('\n'), bold]; + return [title, content, h2.join('\n'), bold.join('\n')]; } /** @@ -92,36 +100,37 @@ function getContent(filename, data) { * @param modified - Array of added/modified files * @param removed - Array of removed files */ -function updateContent(modified, removed) { - const file = String(fs.readFileSync(`/home/ubuntu/OI-wiki/mkdocs.yml`)); +async function updateContent(modified, removed) { + const file = String(fs.readFileSync(path.join(REPO_DIR, 'mkdocs.yml'))); const data = YAML.parse(file.replaceAll('!!python/name:', '')); let ops = []; modified.forEach((filename) => { - ops.push({ index: { _index: 'oiwiki', _id: filename } }); let [title, article, h2, bold] = getContent(filename, data); - ops.push({ - title: title, - content: article, - url: '/' + filename.replace('/index.md', '/').replace('.md', '/'), - h2: h2, - bold: bold, - standard_content: article, - }); + if (title != '') { + ops.push({ index: { _index: 'oiwiki', _id: filename } }); + ops.push({ + title: title, + content: article, + url: '/' + filename.replace('/index.md', '/').replace('.md', '/'), + h2: h2, + bold: bold, + standard_content: article, + }); + } }); removed.forEach((filename) => { ops.push({ delete: { _index: 'oiwiki', _id: filename } }); }); - client.bulk({ body: ops, refresh: 'true' }, function (err, res) { - if (err) { - console.error('Failed Bulk opoeration', err); - res.statusCode = 504; - res.end('elasticsearch bulk op failed'); - return; - } - console.debug('Elasticsearch bulk op success'); - }); + const res = await client.bulk({ body: ops, refresh: 'true' }); + if (res.errors) { + console.error('Bulk operation had errors:', res); + } else { + console.log('Bulk operation succeeded.'); + } } +const handler = createHandler({ path: GITHUB_PATH, secret: GITHUB_SECRET }); + http.createServer((req, res) => { handler(req, res, (err) => { res.statusCode = 404; @@ -134,49 +143,101 @@ handler.on('error', (err) => { console.error('Handler error:', err.message); }); -handler.on('push', (event) => { +handler.on('push', async (event) => { if (event.payload.ref !== 'refs/heads/master') return; console.debug('Received a push event for %s to %s', event.payload.repository.name, event.payload.ref); - exec('bash update.sh', (err) => { - if (err) { - console.error('Shell command execution error:', err); - return; - } - let removed = new Set([]), modified = new Set([]); - event.payload.commits.forEach((e) => { - e.added.forEach(w => { - modified.add(w); - removed.delete(w); - }); - e.modified.forEach(w => modified.add(w)); - e.removed.forEach(w => { - removed.add(w); - modified.delete(w); - }); + await gitWeb.fetch('origin', 'gh-pages'); + await gitWeb.reset(['origin/gh-pages', '--hard']); + console.log('update website'); + + await gitRepo.fetch('gh', 'master'); + await gitRepo.reset(['gh/master', '--hard']); + console.log('update repo'); + + let removed = new Set([]), modified = new Set([]); + event.payload.commits.forEach((e) => { + e.added.forEach(w => { + modified.add(w); + removed.delete(w); + }); + e.modified.forEach(w => modified.add(w)); + e.removed.forEach(w => { + removed.add(w); + modified.delete(w); }); + }); - removed = [...removed].filter(e => e.startsWith('docs') && e.endsWith('md')); - modified = [...modified].filter(e => e.startsWith('docs') && e.endsWith('md')); + removed = [...removed].filter(e => e.startsWith('docs') && e.endsWith('md')); + modified = [...modified].filter(e => e.startsWith('docs') && e.endsWith('md')); - removed.forEach((file, index) => removed[index] = file.replace('docs/', '')); - modified.forEach((file, index) => modified[index] = file.replace('docs/', '')); + removed.forEach((file, index) => removed[index] = file.replace('docs/', '')); + modified.forEach((file, index) => modified[index] = file.replace('docs/', '')); - updateContent(modified, removed); - }); + await updateContent(modified, removed); + console.log('index update'); }); -function init() { - exec(`bash build.sh`, () => { - let modified = []; - const file = String(fs.readFileSync(`/home/ubuntu/OI-wiki/mkdocs.yml`)); - const data = YAML.parse(file.replaceAll('!!python/name:', '')); - traversalArticle(data['nav'], (key, value) => modified.push(value)); - updateContent(modified, []); - }); +/** + * recreate index (delete if exists, then create) + */ +async function recreateIndex() { + const exists = await client.indices.exists({ index: 'oiwiki' }); + if (exists) { + await client.indices.delete({ index: 'oiwiki' }); + console.log('Deleted existing index'); + } + + const body = { + settings: { + analysis: { + analyzer: { + pinyin_analyzer: { + type: 'custom', + tokenizer: 'ik_max_word', + filter: ['pinyin_filter'] + } + }, + filter: { + pinyin_filter: { + type: 'pinyin', + keep_separate_first_letter: true, + keep_full_pinyin: true, + keep_original: true, + first_letter: 'prefix', + limit_first_letter_length: 16, + lowercase: true, + remove_duplicated_term: true + } + } + } + }, + mappings: { + properties: { + content: { type: 'text', analyzer: 'pinyin_analyzer' }, + h2: { type: 'text', analyzer: 'pinyin_analyzer' }, + title: { type: 'text', analyzer: 'pinyin_analyzer' }, + url: { type: 'text' }, + bold: { type: 'text', analyzer: 'pinyin_analyzer' }, + standard_content: { type: 'text', analyzer: 'simple' } + } + } + }; + + await client.indices.create({ index: 'oiwiki', body }); + console.log('Created index'); +} + +async function init() { + await recreateIndex(); + const file = String(fs.readFileSync(path.join(REPO_DIR, 'mkdocs.yml'))); + const data = YAML.parse(file.replaceAll('!!python/name:', '')); + let modified = []; + traversalArticle(data['nav'], (key, value) => modified.push(value)); + await updateContent(modified, []); } init(); From 46cec807588e1586fbe55f434271b6ee8acb6d13 Mon Sep 17 00:00:00 2001 From: ZnPdCo Date: Wed, 24 Sep 2025 14:08:44 +0800 Subject: [PATCH 4/4] refactor --- README.md | 12 ++++++++++++ api/main.js | 3 ++- webhook/index.js | 7 +++---- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index d65d879..fb4ad21 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,13 @@ # elasticsearch-config +这是 OI wiki 的搜索服务器后端，同时能够支持将 build 出来的静态页面发布。 + +`/home/ubuntu/OI-wiki` 是项目源文件储存的地方，用来更新索引；`/var/www/OI-wiki` 是 build 出来的静态文件，用于发布网站。 + +`webhook` 文件夹内的代码会监听 github 上的更新，然后事实更新上面两个文件夹内的仓库，同时更新索引。第一次启动 `webhook` 会清空之前的索引并新建一个。 + +`api` 文件夹内的代码是一个搜索服务器，会在 `localhost:8000` 下启动，api 为 `/?s=manach`。 + 部署需要有 es 环境并安装好 ik 与 pinyin 插件，并将 `/plugins/ik/config/stopword.dic` 与 `/plugins/ik/config/extra_stopword.dic` 内容清空（禁用停用词）： ``` @@ -10,6 +18,10 @@ git remote add gh https://github.com/OI-wiki/OI-wiki.git git clone https://github.com/OI-wiki/OI-wiki.git /var/www/OI-wiki -b gh-pages ``` +直接修改代码或创建环境变量 `GITHUB_PATH` 与 `GITHUB_SECRET` 表示 webhook 的地址与密钥。Github 上的 Content type 配置为 `application/json`。 + +同时也需要修改代码或创建环境变量 `SEARCH_SECRET` 表示搜索服务器的密钥，使用这个密钥可以获取 es 的状态。 + 初始化并启动 webhook： ``` diff --git a/api/main.js b/api/main.js index a743fa8..559b482 100644 --- a/api/main.js +++ b/api/main.js @@ -4,11 +4,12 @@ var client = new elasticsearch.Client({ }); const express = require('express'); const app = express(); +const SEARCH_SECRET = process.env.SEARCH_SECRET || 'SEARCH_SECRET'; app.set('port', process.env.PORT || 8000); app.get('/status', function(req, res) { - if (req.query.s == MY_SECRET_KEY) { + if (req.query.s == SEARCH_SECRET) { client.ping({ requestTimeout: 1000, }, function (error) { diff --git a/webhook/index.js b/webhook/index.js index 9bf42fe..e76a3d8 100644 --- a/webhook/index.js +++ b/webhook/index.js @@ -10,11 +10,10 @@ const math = require('remark-math'); const strip = require('strip-markdown-math'); -// const REPO_DIR = '/home/ubuntu/OI-wiki'; -const REPO_DIR = '/OI-wiki'; +const REPO_DIR = '/home/ubuntu/OI-wiki'; const WEB_DIR = '/var/www/OI-wiki'; -const GITHUB_PATH = process.env.GITHUB_PATH || '/events'; -const GITHUB_SECRET = process.env.GITHUB_SECRET || 'asdlkfijawefojaewiofj'; +const GITHUB_PATH = process.env.GITHUB_PATH || '/GITHUB_PATH'; +const GITHUB_SECRET = process.env.GITHUB_SECRET || 'GITHUB_SECRET'; const gitRepo = simpleGit({ baseDir: REPO_DIR }); const gitWeb = simpleGit({ baseDir: WEB_DIR });