Multilingual Ingestion

# Provider: "llm" (default), "google", or "azure"
TRANSLATION_PROVIDER=llm

# Target language ISO 639-1 code (default: "en")
TARGET_LANGUAGE=en

# Minimum detection confidence to trigger translation (default: 0.8)
CONFIDENCE_THRESHOLD=0.8

import asyncio
import os
import cognee
from cognee.infrastructure.llm import get_max_chunk_tokens
from cognee.tasks.documents import classify_documents, extract_chunks_from_documents
from cognee.shared.data_models import KnowledgeGraph
from cognee.tasks.translation import translate_content
from cognee.modules.pipelines import Task, run_pipeline
from cognee.tasks.graph import extract_graph_from_data
from cognee.tasks.storage import add_data_points

# the translated text is in data_chunks[].text, 
async def drop_translation_metadata(data_chunks):
    for chunk in data_chunks:
        chunk.contains = None
    return data_chunks


async def main():
    await cognee.prune.prune_data()
    await cognee.prune.prune_system(metadata=True)
    text_fr = "La mémoire artificielle permet aux agents IA de retenir des informations complexes."

    tasks = [
        Task(classify_documents),
        Task(extract_chunks_from_documents, max_chunk_size=get_max_chunk_tokens()),
        Task(translate_content, target_language="en", translation_provider="llm"),
        Task(drop_translation_metadata),
        Task(extract_graph_from_data, graph_model=KnowledgeGraph),
        Task(add_data_points),
    ]

    async for _ in run_pipeline(tasks=tasks, datasets=["multilingual"]):
        pass

    visualize_graph_path = os.path.join(
        os.path.dirname(__file__), ".artifacts", "multilingual.html"
    )
    await cognee.visualize_graph(visualize_graph_path)

asyncio.run(main())