compile_base.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

import argparse
import operator
import os
import shutil
from subprocess import call

# Variables
global_version = '0.241.x_p_obf'
protoc_executable = "protoc"
package_name = 'POGOProtos.Rpc'
input_file = "POGOProtos.Rpc.proto"
base_file = os.path.abspath("base/vbase.proto")
protos_path = os.path.abspath("base")
gen_last_files = os.path.abspath("base/last_files")
# args
parser = argparse.ArgumentParser()
parser.add_argument("-gm", "--generate_game_master", help="Generates v2_GAME_MASTER.txt form PATH/v2_GAME_MASTER.")
parser.add_argument("-ga", "--generate_asset_digest", help="Generates ASSET_DIGEST.txt form PATH/ASSET_DIGEST.")
parser.add_argument("-l", "--lang", help="Language to produce proto single file.")
parser.add_argument("-v", "--version", help="Set version out ex:. (0.205.x)")
parser.add_argument("-o", "--out_path", help="Output path for proto single file.")
parser.add_argument("-m", "--java_multiple_files", action='store_true', help='Write each message to a separate .java file.')
parser.add_argument("-g", "--generate_only", action='store_true', help='Generates only proto compilable.')
parser.add_argument("-b", "--generate_new_base", action='store_true', help='Generates new proto base refs.')
parser.add_argument("-k", "--keep_proto_file", action='store_true', help='Do not remove .proto file after compiling.')
parser.add_argument("-gf", "--generate_proto_files", action='store_true', help='Generates base/last_files/*.proto.')
args = parser.parse_args()
# Set defaults args
lang = args.lang or "proto"
out_path = args.out_path or "out/single_file/" + lang
java_multiple_files = args.java_multiple_files
gen_only = args.generate_only
gen_files = args.generate_proto_files
version = args.version or global_version
gen_base = args.generate_new_base
keep_file = args.keep_proto_file
gen_game_master = args.generate_game_master or None
gen_asset_digest = args.generate_asset_digest or None
# Determine where path's and variables
raw_name = "v" + version + ".proto"
raw_proto_file = os.path.abspath("base/" + raw_name)
out_path = os.path.abspath(out_path)
if gen_asset_digest is not None:
    if not os.path.exists(gen_asset_digest):
        print("Binary not found.")
        exit(0)
    commands = []
    print("Try to decode " + gen_asset_digest + ".txt....")
    call(""""{0}" --version""".format(protoc_executable), shell=True)
    pogo_protos_path = './base'
    pogo_protos_target = 'POGOProtos.Rpc.AssetDigestOutProto'
    pogo_protos_template = './base/' + raw_name
    commands.append(
        """"{0}" --proto_path="{1}" --decode "{2}" {3} <{4}> {5}""".format(
            protoc_executable,
            pogo_protos_path,
            pogo_protos_target,
            pogo_protos_template,
            gen_asset_digest,
            gen_asset_digest + '.txt'
        ))
    for command in commands:
        call(command, shell=True)
    exit(0)
if gen_game_master is not None:
    if not os.path.exists(gen_game_master):
        print("Binary not found.")
        exit(0)
    commands = []
    print("Try to decode " + gen_game_master + ".txt....")
    call(""""{0}" --version""".format(protoc_executable), shell=True)
    pogo_protos_path = './base'
    pogo_protos_target = 'POGOProtos.Rpc.DownloadGmTemplatesResponseProto'
    pogo_protos_template = './base/' + raw_name
    commands.append(
        """"{0}" --proto_path="{1}" --decode "{2}" {3} <{4}> {5}""".format(
            protoc_executable,
            pogo_protos_path,
            pogo_protos_target,
            pogo_protos_template,
            gen_game_master,
            gen_game_master + '.txt'
        ))
    for command in commands:
        call(command, shell=True)
    exit(0)
# Add licenses
head = '/*\n'
head += '* Copyright 2016-2022 --=FurtiF=--.\n'
head += '*\n'
head += '* Licensed under the\n'
head += '*	Educational Community License, Version 2.0 (the "License"); you may\n'
head += '*	not use this file except in compliance with the License. You may\n'
head += '*	obtain a copy of the License at\n'
head += '*\n'
head += '*	http://www.osedu.org/licenses/ECL-2.0\n'
head += '*\n'
head += '*	Unless required by applicable law or agreed to in writing,\n'
head += '*	software distributed under the License is distributed on an "AS IS"\n'
head += '*	BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n'
head += '*	or implied. See the License for the specific language governing\n'
head += '*	permissions and limitations under the License.\n'
head += '*\n* Version: ' + version + '\n*\n'
head += '*/\n\n'
head += 'syntax = "proto3";\n'
head += 'package %s;\n\n' % package_name
# Clean up previous out
try:
    os.remove(out_path)
except OSError:
    pass
if out_path and os.path.exists(out_path):
    shutil.rmtree(out_path)
# Create necessary directory
if not os.path.exists(out_path):
    os.makedirs(out_path)
commands = []


def open_proto_file(main_file, head_for_files):
    new_proto_single_file = main_file.replace(raw_name, input_file)
    if os.path.exists(new_proto_single_file):
        os.unlink(new_proto_single_file)
    open_for_new = open(new_proto_single_file, 'a')
    if not gen_only:
        # Add options by language
        if java_multiple_files and lang == "java":
            head_for_files += 'option java_multiple_files = true;\n\n'
        elif lang == "cpp":
            head_for_files += 'option optimize_for = CODE_SIZE;\n\n'
    open_for_new.writelines(head_for_files)
    messages = ''
    # Delete demos
    with open(main_file, 'r') as proto_file:
        for proto_line in proto_file.readlines():
            messages += proto_line
    # check in messages basic obfuscated names...
    proto_name = ''
    for proto_line in messages.split("\n"):
        if operator.contains(proto_line, "{") and len(proto_name) == 11 and proto_name.isupper():
            if operator.contains(proto_line, "oneof "):
                print("OneOf: " + proto_name)
            elif operator.contains(proto_line, "message "):
                print("Message: " + proto_name)
            else:
                print("Enum: " + proto_name)
    # Reorder all this...
    new_base_enums = {}
    new_base_messages = {}
    new_base_as_data = False
    new_base_is_enum = False
    new_base_proto_name = ''
    new_base_data = ''
    for proto_line in messages.split("\n"):
        if proto_line.startswith("enum") or proto_line.startswith("message"):
            new_base_as_data = True
            new_base_proto_name = proto_line.split(" ")[1]
        if proto_line.startswith("enum"):
            new_base_is_enum = True
        if proto_line.startswith("message"):
            new_base_is_enum = False
        if proto_line.startswith("}"):
            new_base_data += proto_line + "\n"
            if new_base_is_enum:
                new_base_enums.setdefault(new_base_proto_name, new_base_data)
            else:
                new_base_messages.setdefault(new_base_proto_name, new_base_data)
            new_base_as_data = False
            new_base_is_enum = False
            new_base_proto_name = ''
            new_base_data = ''
        if new_base_as_data:
            new_base_data += proto_line + "\n"
    new_base_file = ''
    head_file = None
    if gen_files:
        if os.path.exists(gen_last_files):
            shutil.rmtree(gen_last_files)
        if not os.path.exists(gen_last_files):
            os.makedirs(gen_last_files)
        head_file = head_for_files.replace('*\n* Version: ' + version + '\n*\n', '*\n* Note: For references only.\n*\n')
    for p in sorted(new_base_enums):
        new_base_file += new_base_enums[p] + "\n"
        if head_file is not None:
            open_for_new_enum = open(gen_last_files + "/" + p + '.proto', 'a')
            open_for_new_enum.writelines(head_file)
            open_for_new_enum.writelines(new_base_enums[p])
            open_for_new_enum.close()
    for p in sorted(new_base_messages):
        new_base_file += new_base_messages[p] + "\n"
        if head_file is not None:
            open_for_new_message = open(gen_last_files + "/" + p + '.proto', 'a')
            open_for_new_message.writelines(head_file)
            open_for_new_message.writelines(new_base_messages[p])
            open_for_new_message.close()
    # find imports ..
    if head_file is not None:
        for p in sorted(new_base_messages):
            if os.path.exists(gen_last_files + "/" + p + '.proto'):
                os.remove(gen_last_files + "/" + p + '.proto')
            open_for_new_message = open(gen_last_files + "/" + p + '.proto', 'a')
            open_for_new_message.writelines(head_file)
            includes = []
            for message in new_base_messages[p].split("\n"):
                if message.startswith("message") or message.startswith("enum") or message.startswith("}"):
                    continue
                elif message.startswith('\tmap<'):
                    v = message.split("map<")[1].strip().split('>')[0].strip().replace(",", "")
                    v1 = v.split(" ")[0].strip() + '.proto'
                    v2 = v.split(" ")[1].strip() + '.proto'
                    if os.path.exists(gen_last_files + "/" + v1):
                        if v1 not in includes:
                            includes.append(v1)
                    if os.path.exists(gen_last_files + "/" + v2):
                        if v2 not in includes:
                            includes.append(v2)
                elif message.startswith('\trepeated') or message.startswith('\t\trepeated') or message.startswith(
                        '\t\t\trepeated'):
                    file_for_includes = message.split(" ")[1].strip() + '.proto'
                    if os.path.exists(gen_last_files + "/" + file_for_includes) and file_for_includes not in includes:
                        includes.append(file_for_includes)
                else:
                    file_for_includes = message.split(" ")[0].strip() + '.proto'
                    if os.path.exists(gen_last_files + "/" + file_for_includes) and file_for_includes not in includes:
                        includes.append(file_for_includes)
            if len(includes) > 0:
                files_inc = ''
                for file in includes:
                    if file == 'ITEM.proto':
                        continue
                    files_inc += 'import "' + file + '";\n'
                if files_inc != '':
                    open_for_new_message.writelines(files_inc + '\n')
            open_for_new_message.writelines(new_base_messages[p])
            open_for_new_message.close()
    messages = new_base_file
    open_for_new.writelines(messages[:-1])
    open_for_new.close()
    add_command_for_new_proto_file(new_proto_single_file)


def add_command_for_new_proto_file(file):
    command_out_path = os.path.abspath(out_path)
    options = ''
    arguments = ''
    if lang == 'js':
        options = 'import_style=commonjs,binary'
    elif lang == 'swift':
        arguments = '--swift_opt=Visibility=Public'
    # elif lang == 'csharp':
    #    arguments = '--csharp_opt=file_extension=.g.cs --csharp_opt=base_namespace'
    # elif lang == 'dart':
    #    arguments = '--plugin "pub run protoc_plugin"'
    # elif lang == 'lua':
    #    arguments = '--plugin=protoc-gen-lua="../ProtoGenLua/plugin/build.bat"'
    elif lang == 'go':
        options = 'plugins=grpc'
    commands.append(
        """{0} --proto_path={1} --{2}_out={3}:{4} {5} {6}""".format(
            protoc_executable,
            protos_path,
            lang,
            options,
            command_out_path,
            arguments,
            file
        )
    )


compile_ext = 'v' + version + '.proto'
if lang == 'python':
    package_name = package_name.replace('.', '_')
    input_file = package_name + '.proto'
if gen_base and gen_files and gen_only:
    if lang == 'proto':
        lang = 'cpp'
    compile_ext += ', remaster_all()'
if gen_base:
    compile_ext += ', base()'
if gen_only:
    compile_ext += ', generate_new()'
if gen_files:
    compile_ext += ', generate_files()'
compile_ext += ', out_mode(' + lang + ')'
print("Compile: " + compile_ext + ", please await...")
print(package_name + " " + version)
call(""""{0}" --version""".format(protoc_executable), shell=True)
open_proto_file(raw_proto_file, head)
generated_file = raw_proto_file.replace(raw_name, input_file)
descriptor_file = generated_file.replace(".proto", ".desc")
descriptor_file_arguments = ['--include_source_info', '--include_imports']
try:
    os.unlink(descriptor_file)
except OSError:
    pass
commands.append(
    """"{0}" --proto_path="{1}" --descriptor_set_out="{2}" {3} {4}""".format(
        protoc_executable,
        protos_path,
        descriptor_file,
        ' '.join(descriptor_file_arguments),
        generated_file))
if not gen_only and not gen_base and not gen_files and not lang == 'proto':
    # Compile commands
    for command in commands:
        call(command, shell=True)
    # Add new desc version
    descriptor_file = generated_file.replace(".proto", ".desc")
    shutil.move(descriptor_file, out_path)
# Add new proto version
if gen_only:
    shutil.copy(generated_file, protos_path + '/v' + version + '.proto')
# New base for next references names
if gen_base:
    try:
        os.unlink(base_file)
    except OSError:
        pass
    shutil.copy(generated_file, base_file)
if keep_file or lang == "proto":
    shutil.move(generated_file, out_path)

# Clean genererated and unneded files
try:
    os.unlink(generated_file)
except OSError:
    pass
print("Done!")