-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathembed.py
More file actions
25 lines (22 loc) · 794 Bytes
/
Copy pathembed.py
File metadata and controls
25 lines (22 loc) · 794 Bytes
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
print("Running embed.py")
import json
from sentence_transformers import SentenceTransformer
import faiss
import numpy as np
print("Loading courses...")
with open('data/courses.json', 'r') as f:
courses = json.load(f)
print("Loading embedding model...")
model = SentenceTransformer('all-MiniLM-L6-v2')
texts = [f"{c['code']} - {c['title']}: {c['description']} Instructor: {c['instructor']}" for c in courses]
embeddings = model.encode(texts)
print("Creating embeddings...")
dimension = embeddings.shape[1]
index = faiss.IndexFlatL2(dimension)
index.add(np.array(embeddings))
print("Building FAISS index...")
faiss.write_index(index, 'data/course_index.faiss')
print("Saving metadata...")
with open('data/course_index.json', 'w') as f:
json.dump(courses, f)
print("All files saved.")