gazelle/python/parse.py

# Copyright 2023 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# parse.py is a long-living program that communicates over STDIN and STDOUT.
# STDIN receives parse requests, one per line. It outputs the parsed modules and
# comments from all the files from each request.

import ast
import concurrent.futures
import json
import os
import sys
from io import BytesIO
from tokenize import COMMENT, NAME, OP, STRING, tokenize


def parse_import_statements(content, filepath):
    modules = list()
    tree = ast.parse(content, filename=filepath)
    for node in ast.walk(tree):
        if isinstance(node, ast.Import):
            for subnode in node.names:
                module = {
                    "name": subnode.name,
                    "lineno": node.lineno,
                    "filepath": filepath,
                    "from": "",
                }
                modules.append(module)
        elif isinstance(node, ast.ImportFrom) and node.level == 0:
            for subnode in node.names:
                module = {
                    "name": f"{node.module}.{subnode.name}",
                    "lineno": node.lineno,
                    "filepath": filepath,
                    "from": node.module,
                }
                modules.append(module)
    return modules


def parse_comments(content):
    comments = list()
    g = tokenize(BytesIO(content.encode("utf-8")).readline)
    for toknum, tokval, _, _, _ in g:
        if toknum == COMMENT:
            comments.append(tokval)
    return comments


def parse_main(content):
    g = tokenize(BytesIO(content.encode("utf-8")).readline)
    for token_type, token_val, start, _, _ in g:
        if token_type != NAME or token_val != "if" or start[1] != 0:
            continue
        try:
            token_type, token_val, start, _, _ = next(g)
            if token_type != NAME or token_val != "__name__":
                continue
            token_type, token_val, start, _, _ = next(g)
            if token_type != OP or token_val != "==":
                continue
            token_type, token_val, start, _, _ = next(g)
            if token_type != STRING or token_val.strip("\"'") != '__main__':
                continue
            token_type, token_val, start, _, _ = next(g)
            if token_type != OP or token_val != ":":
                continue
            return True
        except StopIteration:
            break
    return False


def parse(repo_root, rel_package_path, filename):
    rel_filepath = os.path.join(rel_package_path, filename)
    abs_filepath = os.path.join(repo_root, rel_filepath)
    with open(abs_filepath, "r") as file:
        content = file.read()
        # From simple benchmarks, 2 workers gave the best performance here.
        with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
            modules_future = executor.submit(
                parse_import_statements, content, rel_filepath
            )
            comments_future = executor.submit(parse_comments, content)
            main_future = executor.submit(parse_main, content)
        modules = modules_future.result()
        comments = comments_future.result()
        has_main = main_future.result()

        output = {
            "filename": filename,
            "modules": modules,
            "comments": comments,
            "has_main": has_main,
        }
        return output


def main(stdin, stdout):
    with concurrent.futures.ProcessPoolExecutor() as executor:
        for parse_request in stdin:
            parse_request = json.loads(parse_request)
            repo_root = parse_request["repo_root"]
            rel_package_path = parse_request["rel_package_path"]
            filenames = parse_request["filenames"]
            outputs = list()
            if len(filenames) == 1:
                outputs.append(parse(repo_root, rel_package_path, filenames[0]))
            else:
                futures = [
                    executor.submit(parse, repo_root, rel_package_path, filename)
                    for filename in filenames
                    if filename != ""
                ]
                for future in concurrent.futures.as_completed(futures):
                    outputs.append(future.result())
            print(json.dumps(outputs), end="", file=stdout, flush=True)
            stdout.buffer.write(bytes([0]))
            stdout.flush()


if __name__ == "__main__":
    exit(main(sys.stdin, sys.stdout))