swr: [rasterizer codegen] Refactor codegen

Move common codegen functions into gen_common.py.

v2: change gen_knobs.py to find the template file internally, like
the rest of the gen scripts.

Reviewed-by: Bruce Cherniak <bruce.cherniak@intel.com>
This commit is contained in:
Tim Rowley 2017-03-17 12:39:15 -05:00
parent caa616ccc4
commit fee3fc018b
7 changed files with 215 additions and 158 deletions

View file

@ -71,30 +71,28 @@ gen_swr_context_llvm.h: rasterizer/codegen/gen_llvm_types.py rasterizer/codegen/
--input $(srcdir)/swr_context.h \
--output ./gen_swr_context_llvm.h
rasterizer/codegen/gen_knobs.cpp: rasterizer/codegen/gen_knobs.py rasterizer/codegen/knob_defs.py rasterizer/codegen/templates/gen_knobs.cpp
rasterizer/codegen/gen_knobs.cpp: rasterizer/codegen/gen_knobs.py rasterizer/codegen/knob_defs.py rasterizer/codegen/templates/gen_knobs.cpp rasterizer/codegen/gen_common.py
$(MKDIR_GEN)
$(PYTHON_GEN) \
$(srcdir)/rasterizer/codegen/gen_knobs.py \
--input $(srcdir)/rasterizer/codegen/templates/gen_knobs.cpp \
--output rasterizer/codegen/gen_knobs.cpp \
--gen_cpp
rasterizer/codegen/gen_knobs.h: rasterizer/codegen/gen_knobs.py rasterizer/codegen/knob_defs.py rasterizer/codegen/templates/gen_knobs.cpp
rasterizer/codegen/gen_knobs.h: rasterizer/codegen/gen_knobs.py rasterizer/codegen/knob_defs.py rasterizer/codegen/templates/gen_knobs.cpp rasterizer/codegen/gen_common.py
$(MKDIR_GEN)
$(PYTHON_GEN) \
$(srcdir)/rasterizer/codegen/gen_knobs.py \
--input $(srcdir)/rasterizer/codegen/templates/gen_knobs.cpp \
--output rasterizer/codegen/gen_knobs.h \
--gen_h
rasterizer/jitter/gen_state_llvm.h: rasterizer/codegen/gen_llvm_types.py rasterizer/codegen/templates/gen_llvm.hpp rasterizer/core/state.h
rasterizer/jitter/gen_state_llvm.h: rasterizer/codegen/gen_llvm_types.py rasterizer/codegen/templates/gen_llvm.hpp rasterizer/core/state.h rasterizer/codegen/gen_common.py
$(MKDIR_GEN)
$(PYTHON_GEN) \
$(srcdir)/rasterizer/codegen/gen_llvm_types.py \
--input $(srcdir)/rasterizer/core/state.h \
--output rasterizer/jitter/gen_state_llvm.h
rasterizer/jitter/gen_builder.hpp: rasterizer/codegen/gen_llvm_ir_macros.py rasterizer/codegen/templates/gen_builder.hpp
rasterizer/jitter/gen_builder.hpp: rasterizer/codegen/gen_llvm_ir_macros.py rasterizer/codegen/templates/gen_builder.hpp rasterizer/codegen/gen_common.py
$(MKDIR_GEN)
$(PYTHON_GEN) \
$(srcdir)/rasterizer/codegen/gen_llvm_ir_macros.py \
@ -102,14 +100,14 @@ rasterizer/jitter/gen_builder.hpp: rasterizer/codegen/gen_llvm_ir_macros.py rast
--output rasterizer/jitter \
--gen_h
rasterizer/jitter/gen_builder_x86.hpp: rasterizer/codegen/gen_llvm_ir_macros.py rasterizer/codegen/templates/gen_builder.hpp
rasterizer/jitter/gen_builder_x86.hpp: rasterizer/codegen/gen_llvm_ir_macros.py rasterizer/codegen/templates/gen_builder.hpp rasterizer/codegen/gen_common.py
$(MKDIR_GEN)
$(PYTHON_GEN) \
$(srcdir)/rasterizer/codegen/gen_llvm_ir_macros.py \
--output rasterizer/jitter \
--gen_x86_h
rasterizer/archrast/gen_ar_event.hpp: rasterizer/codegen/gen_archrast.py rasterizer/codegen/templates/gen_ar_event.hpp rasterizer/archrast/events.proto
rasterizer/archrast/gen_ar_event.hpp: rasterizer/codegen/gen_archrast.py rasterizer/codegen/templates/gen_ar_event.hpp rasterizer/archrast/events.proto rasterizer/codegen/gen_common.py
$(MKDIR_GEN)
$(PYTHON_GEN) \
$(srcdir)/rasterizer/codegen/gen_archrast.py \
@ -117,7 +115,7 @@ rasterizer/archrast/gen_ar_event.hpp: rasterizer/codegen/gen_archrast.py rasteri
--output rasterizer/archrast/gen_ar_event.hpp \
--gen_event_h
rasterizer/archrast/gen_ar_event.cpp: rasterizer/codegen/gen_archrast.py rasterizer/codegen/templates/gen_ar_event.cpp rasterizer/archrast/events.proto
rasterizer/archrast/gen_ar_event.cpp: rasterizer/codegen/gen_archrast.py rasterizer/codegen/templates/gen_ar_event.cpp rasterizer/archrast/events.proto rasterizer/codegen/gen_common.py
$(MKDIR_GEN)
$(PYTHON_GEN) \
$(srcdir)/rasterizer/codegen/gen_archrast.py \
@ -125,7 +123,7 @@ rasterizer/archrast/gen_ar_event.cpp: rasterizer/codegen/gen_archrast.py rasteri
--output rasterizer/archrast/gen_ar_event.cpp \
--gen_event_cpp
rasterizer/archrast/gen_ar_eventhandler.hpp: rasterizer/codegen/gen_archrast.py rasterizer/codegen/templates/gen_ar_eventhandler.hpp rasterizer/archrast/events.proto
rasterizer/archrast/gen_ar_eventhandler.hpp: rasterizer/codegen/gen_archrast.py rasterizer/codegen/templates/gen_ar_eventhandler.hpp rasterizer/archrast/events.proto rasterizer/codegen/gen_common.py
$(MKDIR_GEN)
$(PYTHON_GEN) \
$(srcdir)/rasterizer/codegen/gen_archrast.py \
@ -133,7 +131,7 @@ rasterizer/archrast/gen_ar_eventhandler.hpp: rasterizer/codegen/gen_archrast.py
--output rasterizer/archrast/gen_ar_eventhandler.hpp \
--gen_eventhandler_h
rasterizer/archrast/gen_ar_eventhandlerfile.hpp: rasterizer/codegen/gen_archrast.py rasterizer/codegen/templates/gen_ar_eventhandlerfile.hpp rasterizer/archrast/events.proto
rasterizer/archrast/gen_ar_eventhandlerfile.hpp: rasterizer/codegen/gen_archrast.py rasterizer/codegen/templates/gen_ar_eventhandlerfile.hpp rasterizer/archrast/events.proto rasterizer/codegen/gen_common.py
$(MKDIR_GEN)
$(PYTHON_GEN) \
$(srcdir)/rasterizer/codegen/gen_archrast.py \

View file

@ -24,25 +24,7 @@ from __future__ import print_function
import os
import sys
import re
import argparse
from mako.template import Template
from mako.exceptions import RichTraceback
def write_template_to_string(template_filename, **kwargs):
try:
template = Template(filename=template_filename)
# Split + Join fixes line-endings for whatever platform you are using
return '\n'.join(template.render(**kwargs).splitlines())
except:
traceback = RichTraceback()
for (filename, lineno, function, line) in traceback.traceback:
print('File %s, line %s, in %s' % (filename, lineno, function))
print(line, '\n')
print('%s: %s' % (str(traceback.error.__class__.__name__), traceback.error))
def write_template_to_file(template_filename, output_filename, **kwargs):
with open(output_filename, 'w') as outfile:
print(write_template_to_string(template_filename, **kwargs), file=outfile)
from gen_common import ArgumentParser, MakoTemplateWriter
def parse_event_fields(lines, idx, event_dict):
field_names = []
@ -141,7 +123,7 @@ def parse_protos(filename):
def main():
# Parse args...
parser = argparse.ArgumentParser()
parser = ArgumentParser()
parser.add_argument('--proto', '-p', help='Path to proto file', required=True)
parser.add_argument('--output', '-o', help='Output filename (i.e. event.hpp)', required=True)
parser.add_argument('--gen_event_hpp', help='Generate event header', action='store_true', default=False)
@ -172,7 +154,7 @@ def main():
template_file = os.sep.join([curdir, 'templates', 'gen_ar_event.hpp'])
output_fullpath = os.sep.join([output_dir, output_filename])
write_template_to_file(template_file, output_fullpath,
MakoTemplateWriter.to_file(template_file, output_fullpath,
filename=output_filename,
protos=protos)
@ -182,7 +164,7 @@ def main():
template_file = os.sep.join([curdir, 'templates', 'gen_ar_event.cpp'])
output_fullpath = os.sep.join([output_dir, output_filename])
write_template_to_file(template_file, output_fullpath,
MakoTemplateWriter.to_file(template_file, output_fullpath,
filename=output_filename,
protos=protos)
@ -192,7 +174,7 @@ def main():
template_file = os.sep.join([curdir, 'templates', 'gen_ar_eventhandler.hpp'])
output_fullpath = os.sep.join([output_dir, output_filename])
write_template_to_file(template_file, output_fullpath,
MakoTemplateWriter.to_file(template_file, output_fullpath,
filename=output_filename,
event_header='gen_ar_event.hpp',
protos=protos)
@ -203,7 +185,7 @@ def main():
template_file = os.sep.join([curdir, 'templates', 'gen_ar_eventhandlerfile.hpp'])
output_fullpath = os.sep.join([output_dir, output_filename])
write_template_to_file(template_file, output_fullpath,
MakoTemplateWriter.to_file(template_file, output_fullpath,
filename=output_filename,
event_header='gen_ar_eventhandler.hpp',
protos=protos)

View file

@ -24,36 +24,14 @@
from __future__ import print_function
import itertools
import math
import argparse
import os
import sys
from mako.template import Template
from mako.exceptions import RichTraceback
def write_template_to_string(template_filename, **kwargs):
try:
template = Template(filename=os.path.abspath(template_filename))
# Split + Join fixes line-endings for whatever platform you are using
return '\n'.join(template.render(**kwargs).splitlines())
except:
traceback = RichTraceback()
for (filename, lineno, function, line) in traceback.traceback:
print("File %s, line %s, in %s" % (filename, lineno, function))
print(line, "\n")
print("%s: %s" % (str(traceback.error.__class__.__name__), traceback.error))
def write_template_to_file(template_filename, output_filename, **kwargs):
output_dirname = os.path.dirname(output_filename)
if not os.path.exists(output_dirname):
os.makedirs(output_dirname)
with open(output_filename, "w") as outfile:
print(write_template_to_string(template_filename, **kwargs), file=outfile)
from gen_common import ArgumentParser, MakoTemplateWriter
def main(args=sys.argv[1:]):
thisDir = os.path.dirname(os.path.realpath(__file__))
parser = argparse.ArgumentParser("Generate files and initialization functions for all permutuations of BackendPixelRate.")
parser = ArgumentParser("Generate files and initialization functions for all permutuations of BackendPixelRate.")
parser.add_argument('--dim', help="gBackendPixelRateTable array dimensions", nargs='+', type=int, required=True)
parser.add_argument('--outdir', help="output directory", nargs='?', type=str, default=thisDir)
parser.add_argument('--split', help="how many lines of initialization per file [0=no split]", nargs='?', type=int, default='512')
@ -100,7 +78,7 @@ def main(args=sys.argv[1:]):
for fileNum in range(numFiles):
filename = baseCppName % str(fileNum)
#print('Generating', filename)
write_template_to_file(
MakoTemplateWriter.to_file(
templateCpp,
baseCppName % str(fileNum),
cmdline=sys.argv,
@ -112,7 +90,7 @@ def main(args=sys.argv[1:]):
templateCmake = os.path.join(thisDir, 'templates', 'gen_backend.cmake')
cmakeFile = os.path.join(args.outdir, 'gen_backends.cmake')
#print('Generating', cmakeFile)
write_template_to_file(
MakoTemplateWriter.to_file(
templateCmake,
cmakeFile,
cmdline=sys.argv,

View file

@ -0,0 +1,162 @@
# Copyright (C) 2014-2016 Intel Corporation. All Rights Reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice (including the next
# paragraph) shall be included in all copies or substantial portions of the
# Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
# Python source
from __future__ import print_function
import os
import sys
import argparse
from mako.template import Template
from mako.exceptions import RichTraceback
#==============================================================================
class MakoTemplateWriter:
'''
MakoTemplateWriter - Class (namespace) for functions to generate strings
or files using the Mako template module.
See http://docs.makotemplates.org/en/latest/ for
mako documentation.
'''
@staticmethod
def to_string(template_filename, **kwargs):
'''
Write template data to a string object and return the string
'''
from mako.template import Template
from mako.exceptions import RichTraceback
try:
template = Template(filename=template_filename)
# Split + Join fixes line-endings for whatever platform you are using
return '\n'.join(template.render(**kwargs).splitlines())
except:
traceback = RichTraceback()
for (filename, lineno, function, line) in traceback.traceback:
print('File %s, line %s, in %s' % (filename, lineno, function))
print(line, '\n')
print('%s: %s' % (str(traceback.error.__class__.__name__), traceback.error))
@staticmethod
def to_file(template_filename, output_filename, **kwargs):
'''
Write template data to a file
'''
with open(output_filename, 'w') as outfile:
print(MakoTemplateWriter.to_string(template_filename, **kwargs), file=outfile)
#==============================================================================
class ArgumentParser(argparse.ArgumentParser):
'''
Subclass of argparse.ArgumentParser
Allow parsing from command files that start with @
Example:
>bt run @myargs.txt
Contents of myargs.txt:
-m <machine>
--target cdv_win7
The below function allows multiple args to be placed on the same text-file line.
The default is one token per line, which is a little cumbersome.
Also allow all characters after a '#' character to be ignored.
'''
#==============================================================================
class _HelpFormatter(argparse.RawTextHelpFormatter):
''' Better help formatter for argument parser '''
def _split_lines(self, text, width):
''' optimized split lines algorighm, indents split lines '''
lines = text.splitlines()
out_lines = []
if len(lines):
out_lines.append(lines[0])
for line in lines[1:]:
out_lines.append(' ' + line)
return out_lines
#==============================================================================
def __init__(self, *args, **kwargs):
''' Constructor. Compatible with argparse.ArgumentParser(),
but with some modifications for better usage and help display.
'''
super(ArgumentParser, self).__init__(
*args,
fromfile_prefix_chars='@',
formatter_class=ArgumentParser._HelpFormatter,
**kwargs)
#==========================================================================
def convert_arg_line_to_args(self, arg_line):
''' convert one line of parsed file to arguments '''
arg_line = arg_line.split('#', 1)[0]
if sys.platform == 'win32':
arg_line = arg_line.replace('\\', '\\\\')
for arg in shlex.split(arg_line):
if not arg.strip():
continue
yield arg
#==========================================================================
def _read_args_from_files(self, arg_strings):
''' read arguments from files '''
# expand arguments referencing files
new_arg_strings = []
for arg_string in arg_strings:
# for regular arguments, just add them back into the list
if arg_string[0] not in self.fromfile_prefix_chars:
new_arg_strings.append(arg_string)
# replace arguments referencing files with the file content
else:
filename = arg_string[1:]
# Search in sys.path
if not os.path.exists(filename):
for path in sys.path:
filename = os.path.join(path, arg_string[1:])
if os.path.exists(filename):
break
try:
args_file = open(filename)
try:
arg_strings = []
for arg_line in args_file.read().splitlines():
for arg in self.convert_arg_line_to_args(arg_line):
arg_strings.append(arg)
arg_strings = self._read_args_from_files(arg_strings)
new_arg_strings.extend(arg_strings)
finally:
args_file.close()
except IOError:
err = sys.exc_info()[1]
self.error(str(err))
# return the modified argument list
return new_arg_strings

View file

@ -23,59 +23,41 @@
from __future__ import print_function
import os
import sys
import argparse
import knob_defs
from mako.template import Template
from mako.exceptions import RichTraceback
def write_template_to_string(template_filename, **kwargs):
try:
template = Template(filename=os.path.abspath(template_filename))
# Split + Join fixes line-endings for whatever platform you are using
return '\n'.join(template.render(**kwargs).splitlines())
except:
traceback = RichTraceback()
for (filename, lineno, function, line) in traceback.traceback:
print("File %s, line %s, in %s" % (filename, lineno, function))
print(line, "\n")
print("%s: %s" % (str(traceback.error.__class__.__name__), traceback.error))
def write_template_to_file(template_filename, output_filename, **kwargs):
output_dirname = os.path.dirname(output_filename)
if not os.path.exists(output_dirname):
os.makedirs(output_dirname)
with open(output_filename, "w") as outfile:
print(write_template_to_string(template_filename, **kwargs), file=outfile)
from gen_common import MakoTemplateWriter, ArgumentParser
def main(args=sys.argv[1:]):
# parse args
parser = argparse.ArgumentParser()
parser.add_argument("--input", "-i", help="Path to gen_knobs.cpp template", required=True)
parser = ArgumentParser()
parser.add_argument("--output", "-o", help="Path to output file", required=True)
parser.add_argument("--gen_h", "-gen_h", help="Generate gen_knobs.h", action="store_true", default=False)
parser.add_argument("--gen_cpp", "-gen_cpp", help="Generate gen_knobs.cpp", action="store_true", required=False)
args = parser.parse_args()
if args.input:
if args.gen_h:
write_template_to_file(args.input,
args.output,
cmdline=sys.argv,
filename='gen_knobs',
knobs=knob_defs.KNOBS,
includes=['core/knobs_init.h', 'common/os.h', 'sstream', 'iomanip'],
gen_header=True)
cur_dir = os.path.dirname(os.path.abspath(__file__))
template_file = os.path.join(cur_dir, 'templates', 'gen_knobs.cpp')
if args.gen_cpp:
write_template_to_file(args.input,
args.output,
cmdline=sys.argv,
filename='gen_knobs',
knobs=knob_defs.KNOBS,
includes=['core/knobs_init.h', 'common/os.h', 'sstream', 'iomanip'],
gen_header=False)
if args.gen_h:
MakoTemplateWriter.to_file(
template_file,
args.output,
cmdline=sys.argv,
filename='gen_knobs',
knobs=knob_defs.KNOBS,
includes=['core/knobs_init.h', 'common/os.h', 'sstream', 'iomanip'],
gen_header=True)
if args.gen_cpp:
MakoTemplateWriter.to_file(
template_file,
args.output,
cmdline=sys.argv,
filename='gen_knobs',
knobs=knob_defs.KNOBS,
includes=['core/knobs_init.h', 'common/os.h', 'sstream', 'iomanip'],
gen_header=False)
return 0

View file

@ -21,31 +21,8 @@
from __future__ import print_function
import os, sys, re
import argparse
import json as JSON
import operator
from mako.template import Template
from mako.exceptions import RichTraceback
def write_template_to_string(template_filename, **kwargs):
try:
template = Template(filename=os.path.abspath(template_filename))
# Split + Join fixes line-endings for whatever platform you are using
return '\n'.join(template.render(**kwargs).splitlines())
except:
traceback = RichTraceback()
for (filename, lineno, function, line) in traceback.traceback:
print('File %s, line %s, in %s' % (filename, lineno, function))
print(line, '\n')
print('%s: %s' % (str(traceback.error.__class__.__name__), traceback.error))
def write_template_to_file(template_filename, output_filename, **kwargs):
output_dirname = os.path.dirname(output_filename)
if not os.path.exists(output_dirname):
os.makedirs(output_dirname)
with open(output_filename, 'w') as outfile:
print(write_template_to_string(template_filename, **kwargs), file=outfile)
from gen_common import MakoTemplateWriter, ArgumentParser
from argparse import FileType
inst_aliases = {
'SHUFFLE_VECTOR': 'VSHUFFLE',
@ -215,7 +192,7 @@ def generate_gen_h(functions, output_dir):
'args' : ', '.join(func['arg_names']),
})
write_template_to_file(
MakoTemplateWriter.to_file(
template,
output_filename,
cmdline=sys.argv,
@ -242,7 +219,7 @@ def generate_x86_h(output_dir):
'intrin' : inst[1],
})
write_template_to_file(
MakoTemplateWriter.to_file(
template,
output_filename,
cmdline=sys.argv,
@ -259,8 +236,8 @@ def generate_x86_h(output_dir):
def main():
# Parse args...
parser = argparse.ArgumentParser()
parser.add_argument('--input', '-i', type=argparse.FileType('r'), help='Path to IRBuilder.h', required=False)
parser = ArgumentParser()
parser.add_argument('--input', '-i', type=FileType('r'), help='Path to IRBuilder.h', required=False)
parser.add_argument('--output-dir', '-o', action='store', dest='output', help='Path to output directory', required=True)
parser.add_argument('--gen_h', help='Generate builder_gen.h', action='store_true', default=False)
parser.add_argument('--gen_x86_h', help='Generate x86 intrinsics. No input is needed.', action='store_true', default=False)

View file

@ -21,30 +21,8 @@
from __future__ import print_function
import os, sys, re
import argparse
import json as JSON
import operator
from mako.template import Template
from mako.exceptions import RichTraceback
def write_template_to_string(template_filename, **kwargs):
try:
template = Template(filename=os.path.abspath(template_filename))
# Split + Join fixes line-endings for whatever platform you are using
return '\n'.join(template.render(**kwargs).splitlines())
except:
traceback = RichTraceback()
for (filename, lineno, function, line) in traceback.traceback:
print('File %s, line %s, in %s' % (filename, lineno, function))
print(line, '\n')
print('%s: %s' % (str(traceback.error.__class__.__name__), traceback.error))
def write_template_to_file(template_filename, output_filename, **kwargs):
output_dirname = os.path.dirname(output_filename)
if not os.path.exists(output_dirname):
os.makedirs(output_dirname)
with open(output_filename, 'w') as outfile:
print(write_template_to_string(template_filename, **kwargs), file=outfile)
from gen_common import MakoTemplateWriter, ArgumentParser
from argparse import FileType
'''
'''
@ -300,7 +278,7 @@ def gen_llvm_types(input_file, output_file):
cur_dir = os.path.dirname(os.path.abspath(__file__))
template = os.path.join(cur_dir, 'templates', 'gen_llvm.hpp')
write_template_to_file(
MakoTemplateWriter.to_file(
template,
output_file,
cmdline=sys.argv,
@ -315,8 +293,8 @@ def gen_llvm_types(input_file, output_file):
def main():
# Parse args...
parser = argparse.ArgumentParser()
parser.add_argument('--input', '-i', type=argparse.FileType('r'),
parser = ArgumentParser()
parser.add_argument('--input', '-i', type=FileType('r'),
help='Path to input file containing structs', required=True)
parser.add_argument('--output', '-o', action='store',
help='Path to output file', required=True)