chore: recovery checkpoint - v3.0 API successfully recovered
CRITICAL CHECKPOINT - DO NOT PUSH TO GITHUB Recovery Status: - Successfully recovered brainy.ts from compiled JavaScript - All core v3.0 API methods functional (add, get, update, delete, relate, find, etc.) - Neural subsystem intact (562KB embedded patterns, NLP working) - Augmentation pipeline operational (20+ augmentations) - HNSW clustering system complete - Triple Intelligence compiled (needs constructor fix) - Test suite validates functionality Changes preserved: - 898 files with changes from last 3 days - 144,475 insertions - All augmentation improvements - All test coverage enhancements - Complete v3.0 feature set This is a LOCAL checkpoint only - contains recovered work after corruption incident. Created backup in .backups/brainy-full-20250910-151314.tar.gz Branch: recovery-checkpoint-20250910-151433 Date: Wed Sep 10 03:18:04 PM PDT 2025
This commit is contained in:
parent
f65455fb22
commit
8ff382ca3b
895 changed files with 143654 additions and 28268 deletions
|
|
@ -0,0 +1,249 @@
|
|||
/**
|
||||
* Optimized S3 Search and Pagination
|
||||
* Provides efficient search and pagination capabilities for S3-compatible storage
|
||||
*/
|
||||
import { createModuleLogger } from '../../utils/logger.js';
|
||||
import { getDirectoryPath } from '../baseStorage.js';
|
||||
const logger = createModuleLogger('OptimizedS3Search');
|
||||
/**
|
||||
* Optimized search implementation for S3-compatible storage
|
||||
*/
|
||||
export class OptimizedS3Search {
|
||||
constructor(storage) {
|
||||
this.storage = storage;
|
||||
}
|
||||
/**
|
||||
* Get nouns with optimized pagination and filtering
|
||||
*/
|
||||
async getNounsWithPagination(options = {}) {
|
||||
const limit = options.limit || 100;
|
||||
const cursor = options.cursor;
|
||||
try {
|
||||
// List noun objects with pagination
|
||||
const listResult = await this.storage.listObjectKeys(`${getDirectoryPath('noun', 'vector')}/`, limit * 2, cursor);
|
||||
if (!listResult.keys.length) {
|
||||
return {
|
||||
items: [],
|
||||
hasMore: false
|
||||
};
|
||||
}
|
||||
// Load nouns in parallel batches
|
||||
const nouns = [];
|
||||
const batchSize = 10;
|
||||
for (let i = 0; i < listResult.keys.length && nouns.length < limit; i += batchSize) {
|
||||
const batch = listResult.keys.slice(i, i + batchSize);
|
||||
const batchPromises = batch.map(key => this.storage.getObject(key));
|
||||
const batchResults = await Promise.all(batchPromises);
|
||||
for (const noun of batchResults) {
|
||||
if (!noun)
|
||||
continue;
|
||||
// Apply filters
|
||||
if (options.filter && !(await this.matchesNounFilter(noun, options.filter))) {
|
||||
continue;
|
||||
}
|
||||
nouns.push(noun);
|
||||
if (nouns.length >= limit) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Determine if there are more items
|
||||
const hasMore = listResult.hasMore || nouns.length >= limit;
|
||||
// Set next cursor
|
||||
let nextCursor;
|
||||
if (hasMore && nouns.length > 0) {
|
||||
nextCursor = nouns[nouns.length - 1].id;
|
||||
}
|
||||
return {
|
||||
items: nouns.slice(0, limit),
|
||||
hasMore,
|
||||
nextCursor
|
||||
};
|
||||
}
|
||||
catch (error) {
|
||||
logger.error('Failed to get nouns with pagination:', error);
|
||||
return {
|
||||
items: [],
|
||||
hasMore: false
|
||||
};
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Get verbs with optimized pagination and filtering
|
||||
*/
|
||||
async getVerbsWithPagination(options = {}) {
|
||||
const limit = options.limit || 100;
|
||||
const cursor = options.cursor;
|
||||
try {
|
||||
// List verb objects with pagination
|
||||
const listResult = await this.storage.listObjectKeys(`${getDirectoryPath('verb', 'vector')}/`, limit * 2, cursor);
|
||||
if (!listResult.keys.length) {
|
||||
return {
|
||||
items: [],
|
||||
hasMore: false
|
||||
};
|
||||
}
|
||||
// Load verbs in parallel batches
|
||||
const verbs = [];
|
||||
const batchSize = 10;
|
||||
for (let i = 0; i < listResult.keys.length && verbs.length < limit; i += batchSize) {
|
||||
const batch = listResult.keys.slice(i, i + batchSize);
|
||||
// Load verbs and their metadata in parallel
|
||||
const batchPromises = batch.map(async (key) => {
|
||||
const verbData = await this.storage.getObject(key);
|
||||
if (!verbData)
|
||||
return null;
|
||||
// Get metadata
|
||||
const verbId = key.replace(`${getDirectoryPath('verb', 'vector')}/`, '').replace('.json', '');
|
||||
const metadata = await this.storage.getMetadata(verbId, 'verb');
|
||||
// Combine into GraphVerb
|
||||
return this.combineVerbWithMetadata(verbData, metadata);
|
||||
});
|
||||
const batchResults = await Promise.all(batchPromises);
|
||||
for (const verb of batchResults) {
|
||||
if (!verb)
|
||||
continue;
|
||||
// Apply filters
|
||||
if (options.filter && !this.matchesVerbFilter(verb, options.filter)) {
|
||||
continue;
|
||||
}
|
||||
verbs.push(verb);
|
||||
if (verbs.length >= limit) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Determine if there are more items
|
||||
const hasMore = listResult.hasMore || verbs.length >= limit;
|
||||
// Set next cursor
|
||||
let nextCursor;
|
||||
if (hasMore && verbs.length > 0) {
|
||||
nextCursor = verbs[verbs.length - 1].id;
|
||||
}
|
||||
return {
|
||||
items: verbs.slice(0, limit),
|
||||
hasMore,
|
||||
nextCursor
|
||||
};
|
||||
}
|
||||
catch (error) {
|
||||
logger.error('Failed to get verbs with pagination:', error);
|
||||
return {
|
||||
items: [],
|
||||
hasMore: false
|
||||
};
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Check if a noun matches the filter criteria
|
||||
*/
|
||||
async matchesNounFilter(noun, filter) {
|
||||
// Get metadata for filtering
|
||||
const metadata = await this.storage.getMetadata(noun.id, 'noun');
|
||||
// Filter by noun type
|
||||
if (filter.nounType) {
|
||||
const nounTypes = Array.isArray(filter.nounType) ? filter.nounType : [filter.nounType];
|
||||
const nounType = metadata?.type || metadata?.noun;
|
||||
if (!nounType || !nounTypes.includes(nounType)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// Filter by service
|
||||
if (filter.service) {
|
||||
const services = Array.isArray(filter.service) ? filter.service : [filter.service];
|
||||
if (!metadata?.service || !services.includes(metadata.service)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// Filter by metadata
|
||||
if (filter.metadata) {
|
||||
if (!metadata)
|
||||
return false;
|
||||
for (const [key, value] of Object.entries(filter.metadata)) {
|
||||
if (metadata[key] !== value) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
/**
|
||||
* Check if a verb matches the filter criteria
|
||||
*/
|
||||
matchesVerbFilter(verb, filter) {
|
||||
// Filter by verb type
|
||||
if (filter.verbType) {
|
||||
const verbTypes = Array.isArray(filter.verbType) ? filter.verbType : [filter.verbType];
|
||||
if (!verb.type || !verbTypes.includes(verb.type)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// Filter by source ID
|
||||
if (filter.sourceId) {
|
||||
const sourceIds = Array.isArray(filter.sourceId) ? filter.sourceId : [filter.sourceId];
|
||||
if (!verb.sourceId || !sourceIds.includes(verb.sourceId)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// Filter by target ID
|
||||
if (filter.targetId) {
|
||||
const targetIds = Array.isArray(filter.targetId) ? filter.targetId : [filter.targetId];
|
||||
if (!verb.targetId || !targetIds.includes(verb.targetId)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// Filter by service
|
||||
if (filter.service) {
|
||||
const services = Array.isArray(filter.service) ? filter.service : [filter.service];
|
||||
if (!verb.metadata?.service || !services.includes(verb.metadata.service)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// Filter by metadata
|
||||
if (filter.metadata) {
|
||||
if (!verb.metadata)
|
||||
return false;
|
||||
for (const [key, value] of Object.entries(filter.metadata)) {
|
||||
if (verb.metadata[key] !== value) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
/**
|
||||
* Combine HNSWVerb data with metadata to create GraphVerb
|
||||
*/
|
||||
combineVerbWithMetadata(verbData, metadata) {
|
||||
if (!verbData || !metadata)
|
||||
return null;
|
||||
// Create default timestamp if not present
|
||||
const defaultTimestamp = {
|
||||
seconds: Math.floor(Date.now() / 1000),
|
||||
nanoseconds: (Date.now() % 1000) * 1000000
|
||||
};
|
||||
// Create default createdBy if not present
|
||||
const defaultCreatedBy = {
|
||||
augmentation: 'unknown',
|
||||
version: '1.0'
|
||||
};
|
||||
return {
|
||||
id: verbData.id,
|
||||
vector: verbData.vector,
|
||||
sourceId: metadata.sourceId,
|
||||
targetId: metadata.targetId,
|
||||
source: metadata.source,
|
||||
target: metadata.target,
|
||||
verb: metadata.verb,
|
||||
type: metadata.type,
|
||||
weight: metadata.weight || 1.0,
|
||||
metadata: metadata.metadata || {},
|
||||
createdAt: metadata.createdAt || defaultTimestamp,
|
||||
updatedAt: metadata.updatedAt || defaultTimestamp,
|
||||
createdBy: metadata.createdBy || defaultCreatedBy,
|
||||
data: metadata.data,
|
||||
embedding: verbData.vector
|
||||
};
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=optimizedS3Search.js.map
|
||||
Loading…
Add table
Add a link
Reference in a new issue