Back to the room

Pass 3 of 3 · Deep dive of PHAROSPass 1 · OverviewPass 2 · How it works

Code Observatory / method

_index_batch

codeintel.indexer.CodeIndexer._index_batch

PINNED SOURCE Source and edges from the index at this commit. An execution appears only where one was recorded.

PHAROS at 975a38dMIT license

Around this symbol

Search all symbols

Loading

Calls (8)
SymbolFileResolved by
read_blobgit_source.pyunique name
_decodeindexer.pysame file
_looks_minifiedindexer.pysame file
_mean_vectorindexer.pysame file
_build_rowsindexer.pysame file
_delete_resourcesindexer.pysame file
_embedindexer.pysame file
parse_file__init__.pythrough an import
Called by (1)
SymbolFileResolved by
_syncindexer.pysame file
Inside (1)
SymbolFileResolved by
CodeIndexerindexer.pysame file
codeintel.indexer.CodeIndexer._index_batchbackend/app/codeintel/indexer.py, lines 335–390Open source
    def _index_batch(self, repo_id, ref: RepoRef, checkout: Checkout, entries: list[TreeEntry],                     existing: dict, stats: IndexStats) -> None:        prepared: list[_PreparedChunk] = []        per_resource: dict[int, list[_PreparedChunk]] = defaultdict(list)        resources: list[Resource] = []        replaced: list = []        t = time.monotonic()        for entry in entries:            try:                text = _decode(checkout.read_blob(entry.blob_sha))                if text is None:                    stats.skipped["binary"] += 1                    continue                if _looks_minified(text):                    stats.skipped["minified"] += 1                    continue                parsed = parse_file(entry.path, text)                if parsed is None:                    stats.skipped["not_source"] += 1                    continue            except Exception as exc:  # one bad file never fails the run                stats.files_failed += 1                stats.errors.append({"path": entry.path, "error": f"{type(exc).__name__}: {exc}"[:300]})                logger.warning("Failed to parse %s: %s", entry.path, exc)                continue            if entry.path in existing:                replaced.append(existing[entry.path][0])            resource, chunks = self._build_rows(repo_id, ref, checkout.commit, entry, text, parsed, stats)            resources.append(resource)            per_resource[id(resource)] = chunks            prepared.extend(chunks)        stats.timings_s["parse"] += time.monotonic() - t        t = time.monotonic()        if self.embedder is not None and prepared:            self._embed(prepared, stats)        stats.timings_s["embed"] += time.monotonic() - t        t = time.monotonic()        if replaced:            self._delete_resources(replaced)        for resource in resources:            chunks = per_resource[id(resource)]            vectors = [p.chunk.embedding for p in chunks if p.chunk.embedding]            resource.embedding = _mean_vector(vectors)            self.db.add(resource)            self.db.flush()            for p in chunks:                p.chunk.resource_id = resource.id            self.db.add_all([p.chunk for p in chunks])        self.db.flush()        stats.chunks_written += len(prepared)        stats.files_indexed += len(resources)        stats.timings_s["db_write"] += time.monotonic() - t

What the index recorded

Signature
def _index_batch(self, repo_id, ref: RepoRef, checkout: Checkout, entries: list[TreeEntry], existing: dict, stats: IndexStats) -> None:
File
backend/app/codeintel/indexer.py
Lines
335–390
Parser
python-ast
Git blob
e8fdfa7b5545e821f7649d68cfadd2aae85b89d0
Call sites
19 named, 10 resolved, 9 left unlinked
Outside this subset
2 more edges reach code outside the graph shown here

Read it on GitHub at this commit