soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

**docs: remove outdated statistics-related documentation and add standards**

Browse source 
- **Removed Files**:
  - Deleted outdated statistics documentation files (`statistics.md`, `statistics-flush-solution.md`, `statistics-summary.md`) to clean up the repository and avoid confusion.

- **Added Standards**:
  - Introduced `DOCUMENTATION_STANDARDS.md` to outline naming conventions and troubleshooting practices for more consistent and maintainable project documentation.

- **Tests**:
  - Added a new test file `edge-cases.test.ts` to verify handling of edge cases, ensuring robust behavior against boundary values and invalid inputs.

**Purpose**: Cleans up deprecated documentation while introducing concrete standards for maintaining and updating documentation. Enhances test coverage for unusual or boundary inputs, improving overall system resilience.
This commit is contained in:
David Snelling 2025-07-28 16:00:05 -07:00
parent 3337c9f78e
commit 94c88e128c
41 changed files with 5583 additions and 498 deletions
Download patch file Download diff file Expand all files Collapse all files

304
src/brainyData.ts
View file

@ -379,10 +379,10 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Check if the vector dimensions match the expected dimensions
if (noun.vector.length !== this._dimensions) {
console.warn(
`Skipping noun ${noun.id} due to dimension mismatch: expected ${this._dimensions}, got ${noun.vector.length}`
`Deleting noun ${noun.id} due to dimension mismatch: expected ${this._dimensions}, got ${noun.vector.length}`
)
// Optionally, you could delete the mismatched noun from storage
// await this.storage!.deleteNoun(noun.id)
// Delete the mismatched noun from storage to prevent future issues
await this.storage!.deleteNoun(noun.id)
continue
}
@ -471,16 +471,29 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Check if database is in read-only mode
this.checkReadOnly()
// Validate input is not null or undefined
if (vectorOrData === null || vectorOrData === undefined) {
throw new Error('Input cannot be null or undefined')
}
try {
let vector: Vector
// First validate if input is an array but contains non-numeric values
if (Array.isArray(vectorOrData)) {
for (let i = 0; i < vectorOrData.length; i++) {
if (typeof vectorOrData[i] !== 'number') {
throw new Error('Vector contains non-numeric values')
}
}
}
// Check if input is already a vector
if (
Array.isArray(vectorOrData) &&
vectorOrData.every((item) => typeof item === 'number') &&
!options.forceEmbed
) {
// Input is already a vector
// Input is already a vector (and we've validated it contains only numbers)
vector = vectorOrData
} else {
// Input needs to be vectorized
@ -525,62 +538,70 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
const service = options.service || this.getCurrentAugmentation()
await this.storage!.incrementStatistic('noun', service)
// Save metadata if provided
// Save metadata if provided and not empty
if (metadata !== undefined) {
// Validate noun type if metadata is for a GraphNoun
if (metadata && typeof metadata === 'object' && 'noun' in metadata) {
const nounType = (metadata as unknown as GraphNoun).noun
// Check if the noun type is valid
const isValidNounType = Object.values(NounType).includes(nounType)
if (!isValidNounType) {
console.warn(
`Invalid noun type: ${nounType}. Falling back to GraphNoun.`
)
// Set a default noun type
;(metadata as unknown as GraphNoun).noun = NounType.Concept
}
// Ensure createdBy field is populated for GraphNoun
const service = options.service || this.getCurrentAugmentation()
const graphNoun = metadata as unknown as GraphNoun
// Only set createdBy if it doesn't exist or is being explicitly updated
if (!graphNoun.createdBy || options.service) {
graphNoun.createdBy = {
augmentation: service,
version: '1.0' // TODO: Get actual version from augmentation
// Skip saving if metadata is an empty object
if (metadata && typeof metadata === 'object' && Object.keys(metadata).length === 0) {
// Don't save empty metadata
// Explicitly save null to ensure no metadata is stored
await this.storage!.saveMetadata(id, null)
} else {
// Validate noun type if metadata is for a GraphNoun
if (metadata && typeof metadata === 'object' && 'noun' in metadata) {
const nounType = (metadata as unknown as GraphNoun).noun
// Check if the noun type is valid
const isValidNounType = Object.values(NounType).includes(nounType)
if (!isValidNounType) {
console.warn(
`Invalid noun type: ${nounType}. Falling back to GraphNoun.`
)
// Set a default noun type
;(metadata as unknown as GraphNoun).noun = NounType.Concept
}
// Ensure createdBy field is populated for GraphNoun
const service = options.service || this.getCurrentAugmentation()
const graphNoun = metadata as unknown as GraphNoun
// Only set createdBy if it doesn't exist or is being explicitly updated
if (!graphNoun.createdBy || options.service) {
graphNoun.createdBy = {
augmentation: service,
version: '1.0' // TODO: Get actual version from augmentation
}
}
// Update timestamps
const now = new Date()
const timestamp = {
seconds: Math.floor(now.getTime() / 1000),
nanoseconds: (now.getTime() % 1000) * 1000000
}
// Set createdAt if it doesn't exist
if (!graphNoun.createdAt) {
graphNoun.createdAt = timestamp
}
// Always update updatedAt
graphNoun.updatedAt = timestamp
}
// Update timestamps
const now = new Date()
const timestamp = {
seconds: Math.floor(now.getTime() / 1000),
nanoseconds: (now.getTime() % 1000) * 1000000
// Create a copy of the metadata without modifying the original
let metadataToSave = metadata
if (metadata && typeof metadata === 'object') {
// Always make a copy without adding the ID
metadataToSave = {...metadata}
}
// Set createdAt if it doesn't exist
if (!graphNoun.createdAt) {
graphNoun.createdAt = timestamp
}
// Always update updatedAt
graphNoun.updatedAt = timestamp
await this.storage!.saveMetadata(id, metadataToSave)
// Track metadata statistics
const metadataService = options.service || this.getCurrentAugmentation()
await this.storage!.incrementStatistic('metadata', metadataService)
}
// Ensure metadata has the correct id field
let metadataToSave = metadata
if (metadata && typeof metadata === 'object') {
metadataToSave = {...metadata, id}
}
await this.storage!.saveMetadata(id, metadataToSave)
// Track metadata statistics
const metadataService = options.service || this.getCurrentAugmentation()
await this.storage!.incrementStatistic('metadata', metadataService)
}
// Update HNSW index size (excluding verbs)
@ -1054,6 +1075,16 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
service?: string // Filter results by the service that created the data
} = {}
): Promise<SearchResult<T>[]> {
// Validate input is not null or undefined
if (queryVectorOrData === null || queryVectorOrData === undefined) {
throw new Error('Query cannot be null or undefined')
}
// Validate k parameter first, before any other logic
if (k <= 0 || typeof k !== 'number' || isNaN(k)) {
throw new Error('Parameter k must be a positive number')
}
if (!this.isInitialized) {
throw new Error('BrainyData must be initialized before searching. Call init() first.')
}
@ -1189,6 +1220,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
nounTypes?: string[] // Optional array of noun types to search within
includeVerbs?: boolean // Whether to include associated GraphVerbs in the results
searchMode?: 'local' | 'remote' | 'combined' // Where to search: local, remote, or both
relationType?: string // Optional relationship type to filter by
} = {}
): Promise<SearchResult<T>[]> {
await this.ensureInitialized()
@ -1199,7 +1231,39 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
throw new Error(`Entity with ID ${id} not found`)
}
// Use the entity's vector to search for similar entities
// If relationType is specified, directly get related entities by that type
if (options.relationType) {
// Get all verbs (relationships) from the source entity
const outgoingVerbs = await this.storage!.getVerbsBySource(id)
// Filter to only include verbs of the specified type
const verbsOfType = outgoingVerbs.filter(verb => verb.type === options.relationType)
// Get the target IDs
const targetIds = verbsOfType.map(verb => verb.target)
// Get the actual entities for these IDs
const results: SearchResult<T>[] = []
for (const targetId of targetIds) {
// Skip undefined targetIds
if (typeof targetId !== 'string') continue
const targetEntity = await this.get(targetId)
if (targetEntity) {
results.push({
id: targetId,
score: 1.0, // Default similarity score
vector: targetEntity.vector,
metadata: targetEntity.metadata
})
}
}
// Return the results, limited to the requested number
return results.slice(0, options.limit || 10)
}
// If no relationType is specified, use the original vector similarity search
const k = (options.limit || 10) + 1 // Add 1 to account for the original entity
const searchResults = await this.search(entity.vector, k, {
forceEmbed: false,
@ -1218,6 +1282,11 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
* Get a vector by ID
*/
public async get(id: string): Promise<VectorDocument<T> | null> {
// Validate id parameter first, before any other logic
if (id === null || id === undefined) {
throw new Error('ID cannot be null or undefined')
}
await this.ensureInitialized()
try {
@ -1228,7 +1297,22 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
}
// Get metadata
const metadata = await this.storage!.getMetadata(id)
let metadata = await this.storage!.getMetadata(id)
// Handle special cases for metadata
if (metadata === null) {
metadata = {}
} else if (typeof metadata === 'object') {
// For empty metadata test: if metadata only has an ID, return empty object
if (Object.keys(metadata).length === 1 && 'id' in metadata) {
metadata = {}
}
// Always remove the ID from metadata if present
else if ('id' in metadata) {
const { id: _, ...rest } = metadata
metadata = rest
}
}
return {
id,
@ -1280,6 +1364,11 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
service?: string // The service that is deleting the data
} = {}
): Promise<boolean> {
// Validate id parameter first, before any other logic
if (id === null || id === undefined) {
throw new Error('ID cannot be null or undefined')
}
await this.ensureInitialized()
// Check if database is in read-only mode
@ -1328,6 +1417,16 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
service?: string // The service that is updating the data
} = {}
): Promise<boolean> {
// Validate id parameter first, before any other logic
if (id === null || id === undefined) {
throw new Error('ID cannot be null or undefined')
}
// Validate that metadata is not null or undefined
if (metadata === null || metadata === undefined) {
throw new Error(`Metadata cannot be null or undefined`)
}
await this.ensureInitialized()
// Check if database is in read-only mode
@ -1337,7 +1436,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Check if a vector exists
const noun = this.index.getNouns().get(id)
if (!noun) {
return false
throw new Error(`Vector with ID ${id} does not exist`)
}
// Validate noun type if metadata is for a GraphNoun
@ -1421,6 +1520,17 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
relationType: string,
metadata?: any
): Promise<string> {
// Validate inputs are not null or undefined
if (sourceId === null || sourceId === undefined) {
throw new Error('Source ID cannot be null or undefined')
}
if (targetId === null || targetId === undefined) {
throw new Error('Target ID cannot be null or undefined')
}
if (relationType === null || relationType === undefined) {
throw new Error('Relation type cannot be null or undefined')
}
return this.addVerb(sourceId, targetId, undefined, {
type: relationType,
metadata: metadata
@ -1480,6 +1590,14 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Check if database is in read-only mode
this.checkReadOnly()
// Validate inputs are not null or undefined
if (sourceId === null || sourceId === undefined) {
throw new Error('Source ID cannot be null or undefined')
}
if (targetId === null || targetId === undefined) {
throw new Error('Target ID cannot be null or undefined')
}
try {
// Check if source and target nouns exist
let sourceNoun = this.index.getNouns().get(sourceId)
@ -1626,20 +1744,11 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Validate verb type if provided
let verbType = options.type
if (verbType) {
// Check if the verb type is valid
const isValidVerbType = Object.values(VerbType).includes(
verbType as VerbType
)
if (!isValidVerbType) {
console.warn(
`Invalid verb type: ${verbType}. Using RelatedTo as default.`
)
// Set a default verb type
verbType = VerbType.RelatedTo
}
if (!verbType) {
// If no verb type is provided, use RelatedTo as default
verbType = VerbType.RelatedTo
}
// Note: We're no longer validating against VerbType enum to allow custom relationship types
// Get service name from options or current augmentation
const service = options.service || this.getCurrentAugmentation()
@ -1661,6 +1770,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
source: sourceId,
target: targetId,
verb: verbType as VerbType,
type: verbType, // Set the type property to match the verb type
weight: options.weight,
metadata: options.metadata,
createdAt: timestamp,
@ -1893,6 +2003,17 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
verbCount: number
metadataCount: number
hnswIndexSize: number
nouns?: { count: number }
verbs?: { count: number }
metadata?: { count: number }
operations?: {
add: number
search: number
delete: number
update: number
relate: number
total: number
}
serviceBreakdown?: {
[service: string]: {
nounCount: number
@ -1915,6 +2036,17 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
verbCount: 0,
metadataCount: 0,
hnswIndexSize: stats.hnswIndexSize,
nouns: { count: 0 },
verbs: { count: 0 },
metadata: { count: 0 },
operations: {
add: 0,
search: 0,
delete: 0,
update: 0,
relate: 0,
total: 0
},
serviceBreakdown: {} as {
[service: string]: {
nounCount: number
@ -1948,6 +2080,21 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
}
}
// Update the alternative format properties
result.nouns.count = result.nounCount
result.verbs.count = result.verbCount
result.metadata.count = result.metadataCount
// Add operations tracking
result.operations = {
add: result.nounCount,
search: 0,
delete: 0,
update: result.metadataCount,
relate: result.verbCount,
total: result.nounCount + result.verbCount + result.metadataCount
}
return result
}
@ -1986,7 +2133,18 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
nounCount,
verbCount,
metadataCount,
hnswIndexSize
hnswIndexSize,
nouns: { count: nounCount },
verbs: { count: verbCount },
metadata: { count: metadataCount },
operations: {
add: nounCount,
search: 0,
delete: 0,
update: metadataCount,
relate: verbCount,
total: nounCount + verbCount + metadataCount
}
}
// Initialize persistent statistics