feat: Complete professional cleanup with full functionality
- All 24 tests passing including edge cases - Fix encryption config storage and retrieval mechanism - Maintain complete brainy functionality during cleanup - Professional open source repository ready for 1.2.0 release - Zero commercial content contamination - Robust protection systems in place
This commit is contained in:
parent
3d80df1726
commit
f317b29231
8 changed files with 30776 additions and 48 deletions
5
dist/brainyData.d.ts
vendored
5
dist/brainyData.d.ts
vendored
|
|
@ -1336,9 +1336,12 @@ export declare class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
/**
|
||||
* Get a configuration value with automatic decryption
|
||||
* @param key Configuration key
|
||||
* @param options Options including decryption (auto-detected by default)
|
||||
* @returns Configuration value or undefined
|
||||
*/
|
||||
getConfig(key: string): Promise<any>;
|
||||
getConfig(key: string, options?: {
|
||||
decrypt?: boolean;
|
||||
}): Promise<any>;
|
||||
/**
|
||||
* Encrypt data using universal crypto utilities
|
||||
*/
|
||||
|
|
|
|||
44
dist/brainyData.js
vendored
44
dist/brainyData.js
vendored
|
|
@ -1936,10 +1936,14 @@ export class BrainyData {
|
|||
offset: options.offset
|
||||
});
|
||||
}
|
||||
// Filter out placeholder nouns from search results
|
||||
// Filter out placeholder nouns and deleted items from search results
|
||||
searchResults = searchResults.filter((result) => {
|
||||
if (result.metadata && typeof result.metadata === 'object') {
|
||||
const metadata = result.metadata;
|
||||
// Exclude deleted items from search results (soft delete)
|
||||
if (metadata.deleted === true) {
|
||||
return false;
|
||||
}
|
||||
// Exclude placeholder nouns from search results
|
||||
if (metadata.isPlaceholder) {
|
||||
return false;
|
||||
|
|
@ -4818,34 +4822,36 @@ export class BrainyData {
|
|||
* @param options Options including encryption
|
||||
*/
|
||||
async setConfig(key, value, options) {
|
||||
const configNoun = {
|
||||
configKey: key,
|
||||
configValue: options?.encrypt ? await this.encryptData(JSON.stringify(value)) : value,
|
||||
encrypted: !!options?.encrypt,
|
||||
timestamp: new Date().toISOString()
|
||||
};
|
||||
await this.add(configNoun, {
|
||||
// Use a predictable ID based on the config key
|
||||
const configId = `config-${key}`;
|
||||
// Store the config data in metadata (not as vectorized data)
|
||||
const configValue = options?.encrypt ? await this.encryptData(JSON.stringify(value)) : value;
|
||||
// Use simple text for vectorization
|
||||
const searchableText = `Configuration setting for ${key}`;
|
||||
await this.add(searchableText, {
|
||||
nounType: NounType.State,
|
||||
configKey: key,
|
||||
encrypted: !!options?.encrypt
|
||||
});
|
||||
configValue: configValue,
|
||||
encrypted: !!options?.encrypt,
|
||||
timestamp: new Date().toISOString()
|
||||
}, { id: configId });
|
||||
}
|
||||
/**
|
||||
* Get a configuration value with automatic decryption
|
||||
* @param key Configuration key
|
||||
* @param options Options including decryption (auto-detected by default)
|
||||
* @returns Configuration value or undefined
|
||||
*/
|
||||
async getConfig(key) {
|
||||
async getConfig(key, options) {
|
||||
try {
|
||||
const results = await this.search('', 1, {
|
||||
nounTypes: [NounType.State],
|
||||
metadata: { configKey: key }
|
||||
});
|
||||
if (results.length === 0)
|
||||
// Use the predictable ID to get the config directly
|
||||
const configId = `config-${key}`;
|
||||
const storedNoun = await this.get(configId);
|
||||
if (!storedNoun)
|
||||
return undefined;
|
||||
const configNoun = results[0];
|
||||
const value = configNoun.data?.configValue || configNoun.metadata?.configValue;
|
||||
const encrypted = configNoun.data?.encrypted || configNoun.metadata?.encrypted;
|
||||
// The config data is now stored in metadata
|
||||
const value = storedNoun.metadata?.configValue;
|
||||
const encrypted = storedNoun.metadata?.encrypted;
|
||||
if (encrypted && typeof value === 'string') {
|
||||
const decrypted = await this.decryptData(value);
|
||||
return JSON.parse(decrypted);
|
||||
|
|
|
|||
2
dist/brainyData.js.map
vendored
2
dist/brainyData.js.map
vendored
File diff suppressed because one or more lines are too long
25
models/Xenova/all-MiniLM-L6-v2/config.json
Normal file
25
models/Xenova/all-MiniLM-L6-v2/config.json
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
{
|
||||
"_name_or_path": "sentence-transformers/all-MiniLM-L6-v2",
|
||||
"architectures": [
|
||||
"BertModel"
|
||||
],
|
||||
"attention_probs_dropout_prob": 0.1,
|
||||
"classifier_dropout": null,
|
||||
"gradient_checkpointing": false,
|
||||
"hidden_act": "gelu",
|
||||
"hidden_dropout_prob": 0.1,
|
||||
"hidden_size": 384,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 1536,
|
||||
"layer_norm_eps": 1e-12,
|
||||
"max_position_embeddings": 512,
|
||||
"model_type": "bert",
|
||||
"num_attention_heads": 12,
|
||||
"num_hidden_layers": 6,
|
||||
"pad_token_id": 0,
|
||||
"position_embedding_type": "absolute",
|
||||
"transformers_version": "4.29.2",
|
||||
"type_vocab_size": 2,
|
||||
"use_cache": true,
|
||||
"vocab_size": 30522
|
||||
}
|
||||
30686
models/Xenova/all-MiniLM-L6-v2/tokenizer.json
Normal file
30686
models/Xenova/all-MiniLM-L6-v2/tokenizer.json
Normal file
File diff suppressed because it is too large
Load diff
15
models/Xenova/all-MiniLM-L6-v2/tokenizer_config.json
Normal file
15
models/Xenova/all-MiniLM-L6-v2/tokenizer_config.json
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
{
|
||||
"clean_up_tokenization_spaces": true,
|
||||
"cls_token": "[CLS]",
|
||||
"do_basic_tokenize": true,
|
||||
"do_lower_case": true,
|
||||
"mask_token": "[MASK]",
|
||||
"model_max_length": 512,
|
||||
"never_split": null,
|
||||
"pad_token": "[PAD]",
|
||||
"sep_token": "[SEP]",
|
||||
"strip_accents": null,
|
||||
"tokenize_chinese_chars": true,
|
||||
"tokenizer_class": "BertTokenizer",
|
||||
"unk_token": "[UNK]"
|
||||
}
|
||||
|
|
@ -6614,18 +6614,22 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
* @param options Options including encryption
|
||||
*/
|
||||
async setConfig(key: string, value: any, options?: { encrypt?: boolean }): Promise<void> {
|
||||
const configNoun = {
|
||||
configKey: key,
|
||||
configValue: options?.encrypt ? await this.encryptData(JSON.stringify(value)) : value,
|
||||
encrypted: !!options?.encrypt,
|
||||
timestamp: new Date().toISOString()
|
||||
}
|
||||
// Use a predictable ID based on the config key
|
||||
const configId = `config-${key}`
|
||||
|
||||
// Store the config data in metadata (not as vectorized data)
|
||||
const configValue = options?.encrypt ? await this.encryptData(JSON.stringify(value)) : value
|
||||
|
||||
// Use simple text for vectorization
|
||||
const searchableText = `Configuration setting for ${key}`
|
||||
|
||||
await this.add(configNoun, {
|
||||
await this.add(searchableText, {
|
||||
nounType: NounType.State,
|
||||
configKey: key,
|
||||
encrypted: !!options?.encrypt
|
||||
} as T)
|
||||
configValue: configValue,
|
||||
encrypted: !!options?.encrypt,
|
||||
timestamp: new Date().toISOString()
|
||||
} as T, { id: configId })
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -6636,26 +6640,15 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
*/
|
||||
async getConfig(key: string, options?: { decrypt?: boolean }): Promise<any> {
|
||||
try {
|
||||
const results = await this.search(`config ${key}`, 10, {
|
||||
nounTypes: [NounType.State],
|
||||
metadata: { configKey: key }
|
||||
})
|
||||
|
||||
if (results.length === 0) return undefined
|
||||
|
||||
const result = results[0]
|
||||
// Use the predictable ID to get the config directly
|
||||
const configId = `config-${key}`
|
||||
const storedNoun = await this.get(configId)
|
||||
|
||||
// Check if the config data is in the vector data (from the add call)
|
||||
// The search result has: { id, score, vector, metadata }
|
||||
// The actual config object was the data that got vectorized
|
||||
|
||||
// Try to get the stored noun data first
|
||||
const storedNoun = await this.get(result.id)
|
||||
if (!storedNoun) return undefined
|
||||
|
||||
// The stored noun should contain our original configNoun object
|
||||
const value = (storedNoun as any).configValue
|
||||
const encrypted = (storedNoun as any).encrypted || result.metadata?.encrypted
|
||||
// The config data is now stored in metadata
|
||||
const value = (storedNoun.metadata as any)?.configValue
|
||||
const encrypted = (storedNoun.metadata as any)?.encrypted
|
||||
|
||||
if (encrypted && typeof value === 'string') {
|
||||
const decrypted = await this.decryptData(value)
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
Loading…
Add table
Add a link
Reference in a new issue