From d0edb7161f253449f4ef2fba6328a7b766b5fce6 Mon Sep 17 00:00:00 2001 From: Dmitry Timoshenko Date: Thu, 14 Nov 2024 10:51:24 +0500 Subject: [PATCH 1/5] working version of updated search function, that works for metamodel info of docs, aspects and components --- .gitignore | 2 + instructions.md | 100 ++++++++++ src/frontend/components/Layouts/Menu.vue | 58 ++---- .../components/Search/SearchResults.vue | 177 ++++++++++++++++++ src/frontend/manifest/query.js | 5 + src/frontend/router/index.js | 11 +- src/global/jsonata/queries.mjs | 96 +++++++--- 7 files changed, 380 insertions(+), 69 deletions(-) create mode 100644 instructions.md create mode 100644 src/frontend/components/Search/SearchResults.vue diff --git a/.gitignore b/.gitignore index a7b425f1..ba6342b6 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,8 @@ node_modules /public/workspace/* +.eslintrc.json + # local env files *.env* !example.env diff --git a/instructions.md b/instructions.md new file mode 100644 index 00000000..4643a5b8 --- /dev/null +++ b/instructions.md @@ -0,0 +1,100 @@ +# Global Search Implementation + +## Overview +Implement a global search functionality that allows users to search across all entities in the system including components, aspects, documents, and document content. + +## Requirements + +### UI Requirements +1. Search Input + - Keep existing search input in Menu.vue + - Maintain current search input style + - Trigger search on Enter key or search button click + - Remove menu filtering functionality + +2. Search Results Display + - Show results in main panel + - Use grid view with columns: + - Entity type (with color coding) + - ID + - Title (clickable link) + - Sort results by entity ID + - Use existing grid component styling + - Show "No results found" message when appropriate + - Add "Found in content" indicator for matches in document text + +### Search Functionality +1. Basic Search (Implemented) + - Search across components: + - ID + - Title + - Technologies + - Aspects + - Search across aspects: + - ID + - Title + - Location + - Search across documents: + - ID + - Title/Description + - Subjects + - Location + +2. Content Search (Planned) + - Search within markdown document content + - Use document source field from YAML: + ```yaml + docs: + dochub.manual: + location: DocHub/Руководство + description: Руководство + type: markdown + source: introduction.md + subjects: + - dochub + - dochub.front + ``` + - Index document content during build + - Cache search results for performance + - Show content match indicator in results + +### Technical Requirements +1. Search Implementation + - Use JSONata queries for search + - Implement fuzzy matching for better results + - Handle all entity types consistently + - Ensure proper error handling + - Maintain performance with large datasets + +2. Document Content Search (Future) + - Create search index during build process + - Index markdown content and headers + - Implement efficient content searching + - Consider caching mechanisms + - Add pagination for large result sets + +## Technical Architecture +1. Components + - Menu.vue: Search input + - SearchResults.vue: Results display + - queries.mjs: Search query implementation + +2. Data Flow + - User enters search term + - JSONata query processes search + - Results displayed in grid view + - Links navigate to entities + +## YAML Structure +Documents are defined in YAML files with the following structure: +```yaml +docs: + [document_id]: + location: [navigation path] + description: [display text] + type: [document type] + source: [content file] + subjects: [related items] +``` + +Components and aspects follow their existing YAML structure and are searched using the same mechanism. diff --git a/src/frontend/components/Layouts/Menu.vue b/src/frontend/components/Layouts/Menu.vue index 7c10d4db..07413441 100644 --- a/src/frontend/components/Layouts/Menu.vue +++ b/src/frontend/components/Layouts/Menu.vue @@ -1,8 +1,13 @@ - - - - No results found - - - + @@ -85,23 +137,30 @@ loading: false, searchResults: [], headers: [ - { - text: 'Type', + { + text: 'Тип', value: 'entity', - width: '120px', - sortable: false + width: '120px' }, - { - text: 'ID', - value: 'id', - sortable: true + { + text: 'Заголовок', + value: 'title' }, - { - text: 'Title', - value: 'title', - sortable: true + { + text: 'Релевантность', + value: 'score', + width: '150px' } - ] + ], + entityLabels: { + component: 'Компонент', + aspect: 'Аспект', + document: 'Документ' + }, + noResultsText: 'Результаты не найдены', + matchedInContentText: 'Найдено в содержимом', + relevanceText: 'Релевантность', + currentSearchTerm: '' }; }, watch: { @@ -109,6 +168,7 @@ immediate: true, handler(newQuery) { if (newQuery) { + this.currentSearchTerm = newQuery; this.performSearch(newQuery); } else { this.searchResults = []; @@ -118,34 +178,32 @@ }, methods: { async performSearch(searchQuery) { + console.log('Performing search for:', searchQuery); this.loading = true; - this.searchResults = []; // Clear previous results + this.searchResults = []; try { - console.log('Performing search for:', searchQuery); - const searchExpression = query.search(searchQuery); - console.log('Search expression:', searchExpression); - - const results = await query.expression(searchExpression).evaluate(); - console.log('Raw results:', results); + const results = await query.searchWithContent(searchQuery); + console.log('Raw search results:', results); - // Handle both array and single object results if (results) { - // Convert to array if single object const resultsArray = Array.isArray(results) ? results : [results]; + console.log('Results array:', resultsArray); - // Transform and filter out any null/undefined values this.searchResults = resultsArray - .filter(item => item && item.id) // Only keep valid items + .filter(item => item && item.id) .map(item => ({ entity: String(item.entity || ''), id: String(item.id || ''), title: String(item.title || ''), - link: String(item.link || '') + link: String(item.link || ''), + matchedInContent: Boolean(item.matchedInContent), + contentSnippet: item.contentSnippet, + matchedHeaders: item.matchedHeaders || [], + score: Number(item.score || 0) })); + console.log('Processed search results:', this.searchResults); } - - console.log('Processed search results:', this.searchResults); } catch (error) { console.error('Search error:', error); console.error('Error details:', error.stack); @@ -161,6 +219,25 @@ document: 'info' }; return colors[entity] || 'grey'; + }, + getEntityLabel(entity) { + return this.entityLabels[entity] || entity; + }, + highlightMatches(text) { + if (!text || !this.currentSearchTerm) return text; + + const searchTerms = this.currentSearchTerm.toLowerCase().split(/\s+/); + let highlightedText = text; + + searchTerms.forEach(term => { + const regex = new RegExp(`(${term})`, 'gi'); + highlightedText = highlightedText.replace( + regex, + '$1' + ); + }); + + return highlightedText; } } }; @@ -174,4 +251,37 @@ .v-data-table ::v-deep a { text-decoration: none; } + + .search-result-item { + max-width: 800px; + } + + .content-snippet { + font-size: 0.85em; + line-height: 1.4; + margin: 4px 0; + white-space: pre-line; + overflow: hidden; + text-overflow: ellipsis; + display: -webkit-box; + -webkit-line-clamp: 3; + -webkit-box-orient: vertical; + } + + .matched-headers { + font-size: 0.85em; + } + + ::v-deep .highlight { + background-color: rgba(255, 213, 79, 0.4); + padding: 0 2px; + border-radius: 2px; + } + + .content-match { + background-color: #fafafa; + padding: 8px; + border-radius: 4px; + margin-top: 4px; + } diff --git a/src/frontend/manifest/query.js b/src/frontend/manifest/query.js index c5e5990b..a2793c50 100644 --- a/src/frontend/manifest/query.js +++ b/src/frontend/manifest/query.js @@ -32,7 +32,108 @@ import queries from '@global/jsonata/queries.mjs'; import jsonataFunctions from '@global/jsonata/functions.mjs'; import env from '@front/helpers/env'; import requests from '@front/helpers/requests'; +import Vue from 'vue'; +import Vuex from 'vuex'; +// Initialize Vuex if not already done +Vue.use(Vuex); + +// Create store if it doesn't exist +if (!window.$store) { + window.$store = new Vuex.Store({ + state: { + manifest: { + components: {}, + aspects: {} + } + }, + mutations: { + setManifest(state, manifest) { + state.manifest = manifest; + } + } + }); +} + +// Add index handling +let documentIndex = null; + +// Load index on startup +async function loadDocumentIndex() { + try { + const response = await fetch('/build/document-index.json'); + documentIndex = await response.json(); + console.log('Document index loaded:', Object.keys(documentIndex).length, 'documents'); + } catch (error) { + console.error('Failed to load document index:', error); + documentIndex = {}; + } +} + +// Initialize index +loadDocumentIndex(); + +// Add search functions +const searchFunctions = { + // Get document from index + getDocument: function(id) { + return documentIndex[id]; + }, + + // Search in document content + searchContent: function(text) { + console.log('Searching content for:', text); + const searchText = text.toLowerCase(); + const results = Object.entries(documentIndex || {}) + .filter(([, doc]) => { + const content = doc.content?.toLowerCase() || ''; + const headers = doc.headers?.map(h => h.text.toLowerCase()) || []; + return content.includes(searchText) || headers.some(h => h.includes(searchText)); + }) + .map(([id, doc]) => ({ + id, + title: doc.metadata.description, + entity: 'document', + link: `/docs/${id}`, + score: searchFunctions.calculateScore(text, doc), + matchedInContent: true, + contentSnippet: doc.content.substring(0, 200) + '...' + })); + console.log('Content search results:', results); + return results; + }, + + // Calculate relevance score + calculateScore: function(searchText, doc) { + const text = searchText.toLowerCase(); + let score = 0; + + // Score for title/description match + if (doc.metadata?.description?.toLowerCase().includes(text)) score += 10; + + // Score for headers match + const headerMatches = (doc.headers || []).filter(h => + h.text.toLowerCase().includes(text) + ).length; + score += headerMatches * 5; + + // Score for content match + const contentMatches = (doc.content?.toLowerCase().match(new RegExp(text, 'g')) || []).length; + score += contentMatches; + + // Score for keyword match + if ((doc.keywords || []).some(k => k.toLowerCase().includes(text))) score += 3; + + return score; + } +}; + +// Register functions with JSONata +if (jsonataDriver && typeof jsonataDriver.registerFunction === 'function') { + Object.entries(searchFunctions).forEach(([name, fn]) => { + jsonataDriver.registerFunction(name, fn); + }); +} // Возвращает тело запроса в зависимости от платформы развертывания function resolveJSONataRequest(ID, params) { @@ -114,7 +215,7 @@ const queryDriver = { return resolveJSONataRequest(queries.IDS.DOCUMENTS_FOR_ENTITY, { ENTITY: entity }); }, - // Сводная JSONSchema по всем кастомным сущностям + // Сводна JSONSchema по всем кастомным сущностям entitiesJSONSchema() { return resolveJSONataRequest(queries.IDS.JSONSCEMA_ENTITIES); }, @@ -126,7 +227,91 @@ const queryDriver = { // Add new search query search(searchText) { - return resolveJSONataRequest(queries.IDS.GLOBAL_SEARCH, { SEARCH_TEXT: searchText }); + console.log('Search initiated with:', searchText); + const expression = resolveJSONataRequest(queries.IDS.GLOBAL_SEARCH, { + SEARCH_TEXT: searchText + }); + return this.driver.evaluate(expression); + }, + + // Add new method for content search + async searchWithContent(searchText) { + console.log('Content search initiated with:', searchText); + if (!documentIndex) { + console.log('Loading document index...'); + await loadDocumentIndex(); + } + + // Get store data from Vuex state + const store = window?.Vuex?.state?.manifest; + + // Log store access details for debugging + console.log('Store access details:', { + hasStore: !!store, + componentsCount: store?.components ? Object.keys(store.components).length : 0, + aspectsCount: store?.aspects ? Object.keys(store.aspects).length : 0, + storeData: store // Log full store data + }); + + // First, search in content directly + const contentResults = searchFunctions.searchContent(searchText); + console.log('Content search results:', contentResults); + + // Search in components directly + const componentResults = store?.components ? + Object.entries(store.components) + .filter(([id, comp]) => { + const searchLower = searchText.toLowerCase(); + const titleMatch = comp.title?.toLowerCase().includes(searchLower); + const idMatch = id.toLowerCase().includes(searchLower); + const techMatch = comp.technologies?.some(tech => + tech.toLowerCase().includes(searchLower) + ); + console.log('Component match check:', id, { titleMatch, idMatch, techMatch }); + return titleMatch || idMatch || techMatch; + }) + .map(([id, comp]) => ({ + id, + title: comp.title || id, + entity: 'component', + link: `/entities/components/blank?dh-component-id=${id}`, + score: 5 + })) : []; + + console.log('Component search results:', componentResults); + + // Search in aspects directly + const aspectResults = store?.aspects ? + Object.entries(store.aspects) + .filter(([id, asp]) => { + const searchLower = searchText.toLowerCase(); + const titleMatch = asp.title?.toLowerCase().includes(searchLower); + const idMatch = id.toLowerCase().includes(searchLower); + const locationMatch = asp.location?.toLowerCase().includes(searchLower); + console.log('Aspect match check:', id, { titleMatch, idMatch, locationMatch }); + return titleMatch || idMatch || locationMatch; + }) + .map(([id, asp]) => ({ + id, + title: asp.title || id, + entity: 'aspect', + link: `/entities/aspects/blank?dh-aspect-id=${id}`, + score: 5 + })) : []; + + console.log('Aspect search results:', aspectResults); + + // Combine all results + const allResults = [ + ...componentResults, + ...aspectResults, + ...contentResults + ].filter((item, index, self) => + index === self.findIndex(t => t.id === item.id) + ).sort((a, b) => (b.score || 0) - (a.score || 0)); + + console.log('Final combined results:', allResults); + return allResults; } }; diff --git a/src/global/jsonata/queries.mjs b/src/global/jsonata/queries.mjs index 3435ce25..145d936c 100644 --- a/src/global/jsonata/queries.mjs +++ b/src/global/jsonata/queries.mjs @@ -44,7 +44,8 @@ const IDS = { DOCUMENTS_FOR_ENTITY: QUERY_ID_DOCUMENTS_FOR_ENTITY, JSONSCEMA_ENTITIES: QUERY_ID_JSONSCEMA_ENTITIES, GET_OBJECT: QUERY_GET_OBJECT, - GLOBAL_SEARCH: 'global.search' + GLOBAL_SEARCH: 'global.search', + GLOBAL_SEARCH_WITH_CONTENT: 'global.search.with.content' }; // Then define queries @@ -281,4 +282,80 @@ const queries = { { "id": $ID, "title": $DOC.description, - "entity" \ No newline at end of file + "entity": "document", + "link": "/docs/" & $ID, + "matchedInContent": $fuzzyMatch($CONTENT) // Flag if matched in content + } + ] : [] + ) + ) : []; + + /* Combine results */ + $ALL_RESULTS := ( + $COMPONENTS_AND_ASPECTS := $append($COMPONENTS, $ASPECTS); + $DOCS_ARRAY := $DOCS[0]; + $append($COMPONENTS_AND_ASPECTS, $DOCS_ARRAY) + ); + + /* Sort and return */ + $ALL_RESULTS ? $ALL_RESULTS^(id) : [] + ) + `, + [IDS.GLOBAL_SEARCH_WITH_CONTENT]: ` + ( + /* Get search text parameter */ + $SEARCH := searchText; + $SEARCH_LOWER := $lowercase($SEARCH); + + /* Search in components */ + $COMPONENTS := $each(components, function($value, $key) { + $fuzzyMatch($key) or $fuzzyMatch($value.title) ? { + "id": $key, + "title": $value.title, + "entity": "component", + "link": "/architect/components/" & $key, + "score": 5 + } + }); + + /* Search in aspects */ + $ASPECTS := $each(aspects, function($value, $key) { + $fuzzyMatch($key) or $fuzzyMatch($value.title) ? { + "id": $key, + "title": $value.title, + "entity": "aspect", + "link": "/architect/aspects/" & $key, + "score": 5 + } + }); + + /* Get content results */ + $CONTENT_RESULTS := contentResults; + + /* Combine and sort results */ + $ALL := [ + $COMPONENTS[], + $ASPECTS[], + $CONTENT_RESULTS[] + ]; + + /* Return sorted results or empty array */ + $ALL ? $sort($ALL, function($l, $r) { + $r.score - $l.score + }) : [] + ) + ` +}; + +export default { + IDS, + QUERIES: queries, + // Вставляет в запрос параметры + makeQuery(query, params) { + // eslint-disable-next-line no-useless-escape + return query.replace(/.*(\{\%([A-Z|\_]*)\%\}).*/g, (p1, p2, p3) => { + return `${p1.replace(eval(`/{%${p3}%}/g`), params[p3])}`; + }); + } +}; + From ee5ddaf2f4820d3b0b54eb6fa69058ed9ccd2a78 Mon Sep 17 00:00:00 2001 From: Dmitry Timoshenko Date: Wed, 20 Nov 2024 09:28:53 +0500 Subject: [PATCH 3/5] fully working version --- .dockerignore | 103 ++++---------------------------------------------- Dockerfile | 47 ++++++++--------------- 2 files changed, 23 insertions(+), 127 deletions(-) diff --git a/.dockerignore b/.dockerignore index c16fb37f..bc78819c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,97 +1,8 @@ -# Node -node_modules -npm-debug.log -yarn-debug.log -yarn-error.log - -# Development .git -.gitignore -.env -*.env -!example.env -.vscode -.idea -*.suo -*.ntvs* -*.njsproj -*.sln -*.sw? - -# Project specific -/cache -/dist -/vscode-dist -/certs -/gitlab -/plugins -/src/hidden -/public/temp -/public/workspace -/public/build - -# Documentation and tests -*.md -*.bug -/tests -/docs - -# Build artifacts -*.log -*.pid -*.seed -*.pid.lock - -# Large directories and files -.git/ -.github/ -node_modules/ -**/node_modules/ -**/.git/ -**/*.tar -**/*.gz -**/*.zip -**/*.rar -**/*.7z -**/*.mp4 -**/*.avi -**/*.mov -**/*.wmv -**/*.flv -**/*.iso -**/*.img -**/*.bin -**/*.exe -**/*.dll -**/*.so -**/*.dylib -**/*.jar -**/*.war -**/*.class -**/*.o -**/*.obj -**/*.pdb -**/*.cache -**/*.bak -**/*.swp -**/*.swo -**/*.swn -**/*.DS_Store -**/*.Thumbs.db - -# IDE and editor files -**/*.idea/ -**/*.vscode/ -**/*.sublime-* -**/*.project -**/*.settings -**/*.classpath -**/*.iml -**/*.ipr -**/*.iws - -# Temporary files -**/tmp/ -**/temp/ -**/*.tmp -**/*.temp +.gitlab-ci.yml +Dockerfile +docker-compose*.yaml +node_modules +dist +.github +pics diff --git a/Dockerfile b/Dockerfile index 6911b4aa..78b7499d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,45 +1,30 @@ -# Use multi-stage build -FROM docker/dockerfile:1.3 AS dockerfile-image +ARG NODE_VERSION=20 -FROM node:20-alpine AS deps + + +FROM node:${NODE_VERSION}-alpine AS deps WORKDIR /var/www COPY package.json package-lock.json ./ +COPY plugins ./plugins/ +# RUN --mount=type=cache,target=/root/.npm npm install +RUN npm install -# Create all required plugins with minimal package.json and index.js files -RUN for plugin in html markaper charts smartants plantuml mermaid network asyncapi bpmnjs table drawio devtool svg; do \ - mkdir -p plugins/$plugin && \ - echo "{\"name\": \"$plugin-plugin\", \"version\": \"1.0.0\"}" > plugins/$plugin/package.json && \ - printf "/* eslint-disable */\nexport default {};" > plugins/$plugin/index.js; \ - done -RUN npm install -FROM node:20-alpine AS builder +FROM node:${NODE_VERSION}-alpine AS builder WORKDIR /var/www COPY --from=deps /var/www . COPY . . +COPY --from=deps /var/www/plugins ./plugins/ +ENV NODE_ENV=production +# RUN --mount=type=cache,target=./node_modules/.cache npm run build +RUN npm run build +CMD ["npm", "run", "serve"] +EXPOSE 8080 -# Create necessary directories and copy documentation -RUN mkdir -p public/build && \ - mkdir -p public/documentation/docs/manual/docs && \ - mkdir -p public/documentation/docs/manual/docs/templates && \ - touch public/documentation/docs/manual/docs/templates/empty.md - -# Run indexing and build -RUN npm run build:index && \ - npm run build - -FROM ghcr.io/rabotaru/dochub/nginx:v0.0.3 AS nginx -# Create directories with proper permissions -USER root -RUN mkdir -p /usr/share/nginx/html/build && \ - chown -R nginx:nginx /usr/share/nginx/html -# Copy files -COPY --from=builder /var/www/dist /usr/share/nginx/html -COPY --from=builder /var/www/public/build/document-index.json /usr/share/nginx/html/build/ -# Switch back to nginx user -USER nginx +FROM ghcr.io/rabotaru/dochub/nginx:v0.0.3 as nginx +COPY --chown=101 --from=builder /var/www/dist /usr/share/nginx/html \ No newline at end of file From cb67b9553effe600ded0b24a4af6130800aa1e64 Mon Sep 17 00:00:00 2001 From: Dmitry Timoshenko Date: Wed, 20 Nov 2024 12:28:44 +0500 Subject: [PATCH 4/5] fully working version, with passed check on links for docs, components and aspects --- .../components/Search/SearchResults.vue | 128 +++++++++++------- 1 file changed, 78 insertions(+), 50 deletions(-) diff --git a/src/frontend/components/Search/SearchResults.vue b/src/frontend/components/Search/SearchResults.vue index 5f44529f..bc9bfe8b 100644 --- a/src/frontend/components/Search/SearchResults.vue +++ b/src/frontend/components/Search/SearchResults.vue @@ -1,6 +1,5 @@