Pesquisa híbrida em profundidade para GraphRAG

1
# Instantiate an LLMGraphTransformer object
2
llm_transformer = LLMGraphTransformer(llm=llm)
3
4
# Create a ChatPromptTemplate object
5
template = ChatPromptTemplate.from_messages(
6
    [
7
        (
8
        "system", 
9
        """You are a scientist analyzing scientific journal articles on the effects of various factors, including stress. Your task is to identify and document all entity relationships within these articles, focusing on relationships involving stress as well as those involving other entities. Capture both direct and indirect relationships, exploring connections up to at least five levels deep. Ensure that you only document relationships explicitly mentioned in the articles. Be aware that relationships can be bidirectional. Exclude any nodes related to the author, publisher, and references."""
10
        )
11
    ]
12
)
13
14
# Init the graph transformer, specifying the nodes, relationships and prompt to guide it
15
graph_transformer = LLMGraphTransformer(
16
    llm=llm,
17
    prompt=template
18
)
19
20
# Initialize a all_documents list
21
all_documents = []
22
23
# Process each page individually
24
for page in all_pages:
25
    document = Document(page_content=page_string)
26
    all_documents.append(document)
27
28
# Convert all accumulated documents to graph documents at once
29
graph_docs = graph_transformer.convert_to_graph_documents(all_documents)

1
nodes = {
2
"node1":Node(id='Stress', type='Concept'),
3
"node2":Node(id='Heart Disease', type='Disease'),
4
"node3":Node(id='Burnout', type='Condition'),
5
"node4":Node(id='Mental Health', type='Aspect'),
6
"node5":Node(id='Physical Activity', type='Behavior'),
7
"node6":Node(id='Job Satisfaction', type='Emotional state'),
8
"node7":Node(id='Obesity', type='Condition'),
9
"node8":Node(id='Aging', type='Condition'),
10
#... Redacted for conciseness
11
}
12
13
links = {
14
"rel1":Relationship(source=Node(id='Immune System', type='System'), target=Node(id='Inflammation', type='Condition'), type='REDUCES'),
15
"rel2":Relationship(source=Node(id='Inflammation', type='Condition'), target=Node(id='Heart Disease', type='Disease'), type='CAUSES'),
16
"rel3":Relationship(source=Node(id='Stress', type='Concept'), target=Node(id='Sleep Quality', type='Health aspect'), type='DECREASES'),
17
"rel4":Relationship(source=Node(id='Stress', type='Condition'), target=Node(id='Diet', type='Lifestyle factor'), type='WORSENS')
18
#... Redacted for conciseness
19
}

1
pdf_files = ['Pdf1','pdf2',......'pdfn']
2
for pdf_file in pdf_files:
3
    text = load_pdf(pdf_file)
4
    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=20)
5
    docs = text_splitter.split_documents(text)
6
    MongoDBAtlasVectorSearch.from_documents(docs, OpenAIEmbeddings(), db_config)

1
[
2
    {
3
        '$match': {
4
            '_id': node_name
5
        }
6
    }, {
7
        '$graphLookup': {
8
            'from': 'nodes_relationships', 
9
            'startWith': '$_id', 
10
            'connectFromField': 'relationships', 
11
            'connectToField': '_id', 
12
            'as': 'relates_to', 
13
            'maxDepth': max_depth, 
14
            'depthField': 'distance', 
15
            'restrictSearchWithMatch': {}
16
        }
17
    }
18
    ]

1
question = "How is Social support Related to Aging?"
2
3
k = 3
4
multiplier = 10
5
6
embeddings = embedding_create_backoff(
7
                input=question,
8
                model="text-embedding-ada-002"
9
            )
10
11
query_vector = embeddings.data[0].embedding
12
13
agg_pipeline = [
14
			{
15
	        "$vectorSearch": {
16
	            "index":'vector_index',
17
	            "path": "embedding",
18
	            "queryVector": query_vector,
19
                "limit": k,
20
	            "numCandidates": k * multiplier,
21
	            },
22
	        },
23
	        {
24
    		"$project": {"embedding": 0} 
25
	        }
26
]

1
[{'_id': ObjectId('66e2d52d00e19b410567a4e8'),
2
  'chunks': 'numerous definitions of social support have been examined (e.g., size, \nquality, and intensity of the support network, type of relation, frequency of\nReproduced with permission of the copyright owner.  Further reproduction prohibited without permission.',
3
  'source': './PDF_KG/Stress and Headache.pdf',
4
  'pdf': {'version': '1.10.100',
5
   'info': {'PDFFormatVersion': '1.7',
6
    'IsAcroFormPresent': False,
7
    'IsXFAPresent': False,
8
    'Title': 'Stress and Headache.',
9
    'Author': 'Thomas Hamilton Mosley Jr',
10
    'Creator': 'Appligent AppendPDF Pro 6.3',
11
    'Producer': 'Prince 14.2 (www.princexml.com)',
12
    'CreationDate': "D:20230906103014-07'00'",
13
    'ModDate': "D:20230906103014-07'00'"},
14
   'metadata': {'_metadata': {'dc:format': 'application/pdf',
15
     'dc:creator': 'Thomas Hamilton Mosley Jr',
16
     'dc:title': 'Stress and Headache.',
17
     'pdf:producer': 'Prince 14.2 (www.princexml.com)',
18
     'pdf:appligent': 'AppendPDF Pro 6.3 Linux 64 bit Aug 30 2019 Library 15.0.4',
19
     'xmp:creatortool': 'Appligent AppendPDF Pro 6.3',
20
     'xmp:modifydate': '2023-09-06T10:30:14-07:00',
21
     'xmp:createdate': '2023-09-06T10:30:14-07:00',
22
     'xmp:metadatadate': '2023-09-06T10:30:14-07:00',
23
     'pdfuaid:part': '1',
24
     'xmpmm:documentid': 'uuid:15f91e58-b7e4-11b2-0a00-c04a83010000',
25
     'xmpmm:instanceid': 'uuid:15f91e59-b7e4-11b2-0a00-d0b25ff8ff7f'}},
26
   'totalPages': 201},
27
  'loc': {'pageNumber': 57, 'lines': {'from': 24, 'to': 26}},
28
  'tags': [{'tagName': 'Social Support:Social factor',
29
    'score': 4.098155975341797},
30
   {'tagName': 'Social Support:Behavior', 'score': 2.827536106109619},
31
   {'tagName': 'Social Support:Factor', 'score': 2.827536106109619},
32
   {'tagName': 'Sleep Quality:Health aspect', 'score': 2.2594480514526367},
33
   {'tagName': 'Social Relationships:Condition', 'score': 1.2706201076507568},
34
   {'tagName': 'Social Support:Aspect', 'score': 2.827536106109619},
35
   {'tagName': 'Sleep Quality:Condition', 'score': 2.2594480514526367},
36
   {'tagName': 'Social Support:Activity', 'score': 2.827536106109619},
37
   {'tagName': 'Sleep Quality:Aspect', 'score': 2.2594480514526367},
38
   {'tagName': 'Social Support:Condition', 'score': 2.827536106109619}]},
39
...
40
]

1
[{'_id': 'Social Support:Social factor',
2
  'relates_to': [ {'_id': 'Inflammation:Biological process',
3
                  'distance': 2,
4
                  'relationships': ['Heart Disease:Disease'],
5
                  'targets': {'Heart Disease:Disease': ['CONTRIBUTES_TO']},
6
                  'type': 'Biological process'}],
7
  'relationships': ['Stress:Concept',
8
                    'Mental Health:Condition',
9
                    'Stress:Condition',
10
                    'Mental Health:Health aspect'],
11
  'targets': {'Mental Health:Condition': ['IMPROVES'],
12
              'Mental Health:Health aspect': ['IMPROVES'],
13
              'Stress:Concept': ['REDUCES'],
14
              'Stress:Condition': ['DECREASES', 'REDUCES']},
15
  'type': 'Social factor'},
16
{'_id': 'Exercise:Activity',
17
                  'distance': 2,
18
                  'relationships': ['Physical Health:Condition',
19
                                    'Stress:Concept',
20
                                    'Stress:Condition'],
21
                  'targets': {'Physical Health:Condition': ['IMPROVES'],
22
                              'Stress:Concept': ['REDUCES'],
23
                              'Stress:Condition': ['DECREASES', 'REDUCES']},
24
                  'type': 'Activity'},...]
25
]

1
{'Inflammation:Condition': ['Heart Disease:Disease',
2
              'Diabetes:Disease'],
3
             'Job Satisfaction:Condition': ['Burnout:Condition',
4
              'Productivity:Condition',
5
              'Stress:Condition'],
6
             'Social Support:Condition': ['Mental Health:Condition',
7
              'Stress:Condition'],
8
             'Diet:Behavior': ['Stress:Concept',
9
              'Inflammation:Condition',
10
              'Stress:Condition'],...
11
}

1
{('Inflammation:Condition', 'Heart Disease:Disease'): {'CAUSES',
2
              'CONTRIBUTES',
3
              'CONTRIBUTES_TO',
4
              'INCREASES_RISK'},
5
             ('Inflammation:Condition',
6
              'Diabetes:Disease'): {'CONTRIBUTES_TO'},
7
             ('Job Satisfaction:Condition', 'Burnout:Condition'): {'DECREASES',
8
              'REDUCES'},
9
             ('Job Satisfaction:Condition',
10
              'Productivity:Condition'): {'INCREASES'},
11
             ('Job Satisfaction:Condition', 'Stress:Condition'): {'INFLUENCES',
12
              'REDUCES'},
13
             ('Social Support:Condition',
14
              'Mental Health:Condition'): {'IMPROVES'},
15
             ('Social Support:Condition', 'Stress:Condition'): {'DECREASES',
16
              'REDUCES'},
17
             ('Diet:Behavior', 'Stress:Concept'): {'INFLUENCES'},
18
             ('Diet:Behavior', 'Inflammation:Condition'): {'REDUCES'},
19
             ('Diet:Behavior', 'Stress:Condition'): {'INFLUENCES'},...
20
}

1
chunk = tag_docs[top_id_idx]['chunks']
2
chunk = chunk + f"\n Entity Relationships: {path}"
3
prompt = f'''
4
You are a doctor who specialises in mental health and psychiatry. Please use the below context to answer {question}
5
Please note that the below text has relationship between various entities written after the text Entity Relationships:
6
The relationships are written in comma separated sentences with the relationship names between the entities written in brackets. There can be multiple
7
entity names which are given by slashes. You can choose any of the names to answer the question. Please only answer the question based upon the relationships
8
provided and context given and not from your own previous knowledge. Try and link the relationships with the context, while giving the answer.
9
'''
10
response = chat_completion_backoff(
11
                    model="gpt-3.5-turbo",
12
                    messages=[
13
                        {"role": "system", "content": prompt},
14
                        {"role": "user", "content": chunk},
15
                    ]
16
                )