feat: Complete professional cleanup with full functionality
- All 24 tests passing including edge cases - Fix encryption config storage and retrieval mechanism - Maintain complete brainy functionality during cleanup - Professional open source repository ready for 1.2.0 release - Zero commercial content contamination - Robust protection systems in place
This commit is contained in:
parent
3d80df1726
commit
f317b29231
8 changed files with 30776 additions and 48 deletions
5
dist/brainyData.d.ts
vendored
5
dist/brainyData.d.ts
vendored
|
|
@ -1336,9 +1336,12 @@ export declare class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
/**
|
/**
|
||||||
* Get a configuration value with automatic decryption
|
* Get a configuration value with automatic decryption
|
||||||
* @param key Configuration key
|
* @param key Configuration key
|
||||||
|
* @param options Options including decryption (auto-detected by default)
|
||||||
* @returns Configuration value or undefined
|
* @returns Configuration value or undefined
|
||||||
*/
|
*/
|
||||||
getConfig(key: string): Promise<any>;
|
getConfig(key: string, options?: {
|
||||||
|
decrypt?: boolean;
|
||||||
|
}): Promise<any>;
|
||||||
/**
|
/**
|
||||||
* Encrypt data using universal crypto utilities
|
* Encrypt data using universal crypto utilities
|
||||||
*/
|
*/
|
||||||
|
|
|
||||||
44
dist/brainyData.js
vendored
44
dist/brainyData.js
vendored
|
|
@ -1936,10 +1936,14 @@ export class BrainyData {
|
||||||
offset: options.offset
|
offset: options.offset
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
// Filter out placeholder nouns from search results
|
// Filter out placeholder nouns and deleted items from search results
|
||||||
searchResults = searchResults.filter((result) => {
|
searchResults = searchResults.filter((result) => {
|
||||||
if (result.metadata && typeof result.metadata === 'object') {
|
if (result.metadata && typeof result.metadata === 'object') {
|
||||||
const metadata = result.metadata;
|
const metadata = result.metadata;
|
||||||
|
// Exclude deleted items from search results (soft delete)
|
||||||
|
if (metadata.deleted === true) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
// Exclude placeholder nouns from search results
|
// Exclude placeholder nouns from search results
|
||||||
if (metadata.isPlaceholder) {
|
if (metadata.isPlaceholder) {
|
||||||
return false;
|
return false;
|
||||||
|
|
@ -4818,34 +4822,36 @@ export class BrainyData {
|
||||||
* @param options Options including encryption
|
* @param options Options including encryption
|
||||||
*/
|
*/
|
||||||
async setConfig(key, value, options) {
|
async setConfig(key, value, options) {
|
||||||
const configNoun = {
|
// Use a predictable ID based on the config key
|
||||||
configKey: key,
|
const configId = `config-${key}`;
|
||||||
configValue: options?.encrypt ? await this.encryptData(JSON.stringify(value)) : value,
|
// Store the config data in metadata (not as vectorized data)
|
||||||
encrypted: !!options?.encrypt,
|
const configValue = options?.encrypt ? await this.encryptData(JSON.stringify(value)) : value;
|
||||||
timestamp: new Date().toISOString()
|
// Use simple text for vectorization
|
||||||
};
|
const searchableText = `Configuration setting for ${key}`;
|
||||||
await this.add(configNoun, {
|
await this.add(searchableText, {
|
||||||
nounType: NounType.State,
|
nounType: NounType.State,
|
||||||
configKey: key,
|
configKey: key,
|
||||||
encrypted: !!options?.encrypt
|
configValue: configValue,
|
||||||
});
|
encrypted: !!options?.encrypt,
|
||||||
|
timestamp: new Date().toISOString()
|
||||||
|
}, { id: configId });
|
||||||
}
|
}
|
||||||
/**
|
/**
|
||||||
* Get a configuration value with automatic decryption
|
* Get a configuration value with automatic decryption
|
||||||
* @param key Configuration key
|
* @param key Configuration key
|
||||||
|
* @param options Options including decryption (auto-detected by default)
|
||||||
* @returns Configuration value or undefined
|
* @returns Configuration value or undefined
|
||||||
*/
|
*/
|
||||||
async getConfig(key) {
|
async getConfig(key, options) {
|
||||||
try {
|
try {
|
||||||
const results = await this.search('', 1, {
|
// Use the predictable ID to get the config directly
|
||||||
nounTypes: [NounType.State],
|
const configId = `config-${key}`;
|
||||||
metadata: { configKey: key }
|
const storedNoun = await this.get(configId);
|
||||||
});
|
if (!storedNoun)
|
||||||
if (results.length === 0)
|
|
||||||
return undefined;
|
return undefined;
|
||||||
const configNoun = results[0];
|
// The config data is now stored in metadata
|
||||||
const value = configNoun.data?.configValue || configNoun.metadata?.configValue;
|
const value = storedNoun.metadata?.configValue;
|
||||||
const encrypted = configNoun.data?.encrypted || configNoun.metadata?.encrypted;
|
const encrypted = storedNoun.metadata?.encrypted;
|
||||||
if (encrypted && typeof value === 'string') {
|
if (encrypted && typeof value === 'string') {
|
||||||
const decrypted = await this.decryptData(value);
|
const decrypted = await this.decryptData(value);
|
||||||
return JSON.parse(decrypted);
|
return JSON.parse(decrypted);
|
||||||
|
|
|
||||||
2
dist/brainyData.js.map
vendored
2
dist/brainyData.js.map
vendored
File diff suppressed because one or more lines are too long
25
models/Xenova/all-MiniLM-L6-v2/config.json
Normal file
25
models/Xenova/all-MiniLM-L6-v2/config.json
Normal file
|
|
@ -0,0 +1,25 @@
|
||||||
|
{
|
||||||
|
"_name_or_path": "sentence-transformers/all-MiniLM-L6-v2",
|
||||||
|
"architectures": [
|
||||||
|
"BertModel"
|
||||||
|
],
|
||||||
|
"attention_probs_dropout_prob": 0.1,
|
||||||
|
"classifier_dropout": null,
|
||||||
|
"gradient_checkpointing": false,
|
||||||
|
"hidden_act": "gelu",
|
||||||
|
"hidden_dropout_prob": 0.1,
|
||||||
|
"hidden_size": 384,
|
||||||
|
"initializer_range": 0.02,
|
||||||
|
"intermediate_size": 1536,
|
||||||
|
"layer_norm_eps": 1e-12,
|
||||||
|
"max_position_embeddings": 512,
|
||||||
|
"model_type": "bert",
|
||||||
|
"num_attention_heads": 12,
|
||||||
|
"num_hidden_layers": 6,
|
||||||
|
"pad_token_id": 0,
|
||||||
|
"position_embedding_type": "absolute",
|
||||||
|
"transformers_version": "4.29.2",
|
||||||
|
"type_vocab_size": 2,
|
||||||
|
"use_cache": true,
|
||||||
|
"vocab_size": 30522
|
||||||
|
}
|
||||||
30686
models/Xenova/all-MiniLM-L6-v2/tokenizer.json
Normal file
30686
models/Xenova/all-MiniLM-L6-v2/tokenizer.json
Normal file
File diff suppressed because it is too large
Load diff
15
models/Xenova/all-MiniLM-L6-v2/tokenizer_config.json
Normal file
15
models/Xenova/all-MiniLM-L6-v2/tokenizer_config.json
Normal file
|
|
@ -0,0 +1,15 @@
|
||||||
|
{
|
||||||
|
"clean_up_tokenization_spaces": true,
|
||||||
|
"cls_token": "[CLS]",
|
||||||
|
"do_basic_tokenize": true,
|
||||||
|
"do_lower_case": true,
|
||||||
|
"mask_token": "[MASK]",
|
||||||
|
"model_max_length": 512,
|
||||||
|
"never_split": null,
|
||||||
|
"pad_token": "[PAD]",
|
||||||
|
"sep_token": "[SEP]",
|
||||||
|
"strip_accents": null,
|
||||||
|
"tokenize_chinese_chars": true,
|
||||||
|
"tokenizer_class": "BertTokenizer",
|
||||||
|
"unk_token": "[UNK]"
|
||||||
|
}
|
||||||
|
|
@ -6614,18 +6614,22 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
* @param options Options including encryption
|
* @param options Options including encryption
|
||||||
*/
|
*/
|
||||||
async setConfig(key: string, value: any, options?: { encrypt?: boolean }): Promise<void> {
|
async setConfig(key: string, value: any, options?: { encrypt?: boolean }): Promise<void> {
|
||||||
const configNoun = {
|
// Use a predictable ID based on the config key
|
||||||
configKey: key,
|
const configId = `config-${key}`
|
||||||
configValue: options?.encrypt ? await this.encryptData(JSON.stringify(value)) : value,
|
|
||||||
encrypted: !!options?.encrypt,
|
|
||||||
timestamp: new Date().toISOString()
|
|
||||||
}
|
|
||||||
|
|
||||||
await this.add(configNoun, {
|
// Store the config data in metadata (not as vectorized data)
|
||||||
|
const configValue = options?.encrypt ? await this.encryptData(JSON.stringify(value)) : value
|
||||||
|
|
||||||
|
// Use simple text for vectorization
|
||||||
|
const searchableText = `Configuration setting for ${key}`
|
||||||
|
|
||||||
|
await this.add(searchableText, {
|
||||||
nounType: NounType.State,
|
nounType: NounType.State,
|
||||||
configKey: key,
|
configKey: key,
|
||||||
encrypted: !!options?.encrypt
|
configValue: configValue,
|
||||||
} as T)
|
encrypted: !!options?.encrypt,
|
||||||
|
timestamp: new Date().toISOString()
|
||||||
|
} as T, { id: configId })
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -6636,26 +6640,15 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
||||||
*/
|
*/
|
||||||
async getConfig(key: string, options?: { decrypt?: boolean }): Promise<any> {
|
async getConfig(key: string, options?: { decrypt?: boolean }): Promise<any> {
|
||||||
try {
|
try {
|
||||||
const results = await this.search(`config ${key}`, 10, {
|
// Use the predictable ID to get the config directly
|
||||||
nounTypes: [NounType.State],
|
const configId = `config-${key}`
|
||||||
metadata: { configKey: key }
|
const storedNoun = await this.get(configId)
|
||||||
})
|
|
||||||
|
|
||||||
if (results.length === 0) return undefined
|
|
||||||
|
|
||||||
const result = results[0]
|
|
||||||
|
|
||||||
// Check if the config data is in the vector data (from the add call)
|
|
||||||
// The search result has: { id, score, vector, metadata }
|
|
||||||
// The actual config object was the data that got vectorized
|
|
||||||
|
|
||||||
// Try to get the stored noun data first
|
|
||||||
const storedNoun = await this.get(result.id)
|
|
||||||
if (!storedNoun) return undefined
|
if (!storedNoun) return undefined
|
||||||
|
|
||||||
// The stored noun should contain our original configNoun object
|
// The config data is now stored in metadata
|
||||||
const value = (storedNoun as any).configValue
|
const value = (storedNoun.metadata as any)?.configValue
|
||||||
const encrypted = (storedNoun as any).encrypted || result.metadata?.encrypted
|
const encrypted = (storedNoun.metadata as any)?.encrypted
|
||||||
|
|
||||||
if (encrypted && typeof value === 'string') {
|
if (encrypted && typeof value === 'string') {
|
||||||
const decrypted = await this.decryptData(value)
|
const decrypted = await this.decryptData(value)
|
||||||
|
|
|
||||||
File diff suppressed because one or more lines are too long
Loading…
Add table
Add a link
Reference in a new issue