port to vitis

hls4ml/backends/vitis/vitis_backend.py

@@ -3,6 +3,9 @@
 
 from hls4ml.backends import VivadoBackend
 from hls4ml.model.flow import get_flow, register_flow
+from hls4ml.model.layers import MultiHeadAttention
+from hls4ml.model.optimizer import layer_optimizer
+from hls4ml.model.types import FixedPrecisionType, IntegerPrecisionType, NamedType
 from hls4ml.report import parse_vivado_report
 
 
@@ -13,6 +16,9 @@ def __init__(self):
         self._register_flows()
 
     def _register_flows(self):
+        initializers = self._get_layer_initializers()
+        init_flow = register_flow('init_layers', initializers, requires=['optimize'], backend=self.name)
+
         validation_passes = [
             'vitis:validate_conv_implementation',
             'vitis:validate_resource_strategy',
@@ -30,6 +36,7 @@ def _register_flows(self):
 
         ip_flow_requirements = get_flow('vivado:ip').requires.copy()
         ip_flow_requirements.insert(ip_flow_requirements.index('vivado:init_layers'), validation_flow)
+        ip_flow_requirements.insert(ip_flow_requirements.index('vivado:streaming'), init_flow)
         ip_flow_requirements.insert(ip_flow_requirements.index('vivado:apply_templates'), template_flow)
 
         self._default_flow = register_flow('ip', None, requires=ip_flow_requirements, backend=self.name)
@@ -93,3 +100,24 @@ def build(self, model, reset=False, csim=True, synth=True, cosim=False, validati
         os.chdir(curr_dir)
 
         return parse_vivado_report(model.config.get_output_dir())
+
+    @layer_optimizer(MultiHeadAttention)
+    def init_mha(self, layer):
+        # TODO Allow getting recurrent reuse factor from the config
+        reuse_factor = layer.model.config.get_reuse_factor(layer)
+        layer.set_attr('reuse_factor', reuse_factor)
+        index_t = IntegerPrecisionType(width=1, signed=False)
+        layer.set_attr('index_t', index_t)
+        if 'table_t' not in layer.attributes:
+            layer.set_attr(
+                'table_t', NamedType(name=layer.name + '_table_t', precision=FixedPrecisionType(width=24, integer=8))
+            )
+        if 'table_size' not in layer.attributes:
+            layer.set_attr('table_size', 2048)
+        if 'accum_t' not in layer.attributes:
+            layer.set_attr('accum_t', FixedPrecisionType(width=24, integer=8))
+        if 'inv_range' not in layer.attributes:
+            layer.set_attr('inv_range', 128)
+        if 'exp_range' not in layer.attributes:
+            layer.set_attr('exp_range', 8)
+        layer.set_attr('strategy', 'resource')  # latency

hls4ml/backends/vivado/vivado_backend.py

@@ -21,7 +21,6 @@
     GarNet,
     GarNetStack,
     Layer,
-    MultiHeadAttention,
     Pooling1D,
     Pooling2D,
     SeparableConv1D,
@@ -32,7 +31,6 @@
 from hls4ml.model.optimizer import get_backend_passes, layer_optimizer
 from hls4ml.model.types import FixedPrecisionType, IntegerPrecisionType, NamedType, PackedType
 from hls4ml.report import parse_vivado_report
-from hls4ml.utils import attribute_descriptions as descriptions
 
 
 class VivadoBackend(FPGABackend):
@@ -51,12 +49,10 @@ def _register_layer_attributes(self):
 
         for layer in rnn_layers:
             attrs = self.attribute_map.get(layer, [])
-            attrs.append(ConfigurableAttribute('recurrent_reuse_factor', default=1, description=descriptions.reuse_factor))
-            attrs.append(
-                ConfigurableAttribute('static', value_type=bool, default=True, description=descriptions.recurrent_static)
-            )
-            attrs.append(ConfigurableAttribute('table_size', default=1024, description=descriptions.table_size))
-            attrs.append(TypeAttribute('table', default=FixedPrecisionType(18, 8), description=descriptions.table_type))
+            attrs.append(ConfigurableAttribute('recurrent_reuse_factor', default=1))
+            attrs.append(ConfigurableAttribute('static', value_type=bool, default=True))
+            attrs.append(ConfigurableAttribute('table_size', default=1024))
+            attrs.append(TypeAttribute('table', default=FixedPrecisionType(18, 8)))
             self.attribute_map[layer] = attrs
 
         # Add ParallelizationFactor to Conv1D/2D
@@ -67,29 +63,22 @@ def _register_layer_attributes(self):
 
         for layer in pf_layers:
             attrs = self.attribute_map.get(layer, [])
-            attrs.append(ConfigurableAttribute('parallelization_factor', default=1, description=descriptions.conv_pf))
+            attrs.append(ConfigurableAttribute('parallelization_factor', default=1))
             self.attribute_map[layer] = attrs
 
         # Add ConvImplementation to Convolution+Pooling layers
         cnn_layers = [Conv1D, Conv2D, SeparableConv1D, SeparableConv2D, DepthwiseConv2D, Pooling1D, Pooling2D]
         for layer in cnn_layers:
             attrs = self.attribute_map.get(layer, [])
-            attrs.append(
-                ChoiceAttribute(
-                    'conv_implementation',
-                    choices=['LineBuffer', 'Encoded'],
-                    default='LineBuffer',
-                    description=descriptions.conv_implementation,
-                )
-            )
+            # attrs.append(ConfigurableAttribute('conv_implementation', value_type=str, default='LineBuffer'))
+            attrs.append(ChoiceAttribute('conv_implementation', choices=['LineBuffer', 'Encoded'], default='LineBuffer'))
             self.attribute_map[layer] = attrs
 
     def _register_flows(self):
         initializers = self._get_layer_initializers()
         init_flow = register_flow('init_layers', initializers, requires=['optimize'], backend=self.name)
 
         streaming_passes = [
-            'vivado:inplace_stream_flatten',  # Inform downstream changed packsize in case of skipping flatten
             'vivado:reshape_stream',
             'vivado:clone_output',
             'vivado:insert_zero_padding_before_conv1d',
@@ -124,7 +113,6 @@ def _register_flows(self):
             'vivado:generate_conv_streaming_instructions',
             'vivado:apply_resource_strategy',
             'vivado:generate_conv_im2col',
-            'vivado:generate_pointwise_conv1_d',
             'vivado:generate_unrolled_dense_resource',
             'vivado:set_pipeline_style',
         ]
@@ -661,24 +649,3 @@ def init_garnet(self, layer):
     @layer_optimizer(GarNetStack)
     def init_garnet_stack(self, layer):
         self.init_garnet(layer)
-
-    @layer_optimizer(MultiHeadAttention)
-    def init_mha(self, layer):
-        # TODO Allow getting recurrent reuse factor from the config
-        reuse_factor = layer.model.config.get_reuse_factor(layer)
-        layer.set_attr('reuse_factor', reuse_factor)
-        index_t = IntegerPrecisionType(width=1, signed=False)
-        layer.set_attr('index_t', index_t)
-        if 'table_t' not in layer.attributes:
-            layer.set_attr(
-                'table_t', NamedType(name=layer.name + '_table_t', precision=FixedPrecisionType(width=24, integer=8))
-            )
-        if 'table_size' not in layer.attributes:
-            layer.set_attr('table_size', 2048)
-        if 'accum_t' not in layer.attributes:
-            layer.set_attr('accum_t', FixedPrecisionType(width=24, integer=8))
-        if 'inv_range' not in layer.attributes:
-            layer.set_attr('inv_range', 128)
-        if 'exp_range' not in layer.attributes:
-            layer.set_attr('exp_range', 8)
-        layer.set_attr('strategy', 'resource')  # latency

hls4ml/templates/vivado/nnet_utils/nnet_multiheadattention.h → hls4ml/templates/vitis/nnet_utils/nnet_multiheadattention.h

@@ -6,7 +6,6 @@
 #include "nnet_common.h"
 #include "nnet_dense.h"
 #include "nnet_mult.h"
-#include <iostream>
 #include <math.h>
 
 namespace nnet {

test/pytest/test_multiheadattention.py

@@ -37,12 +37,12 @@ def model():
     return model
 
 
-# Currently only Vivado in io_parallel mode is supported
+# Currently only Vitis in io_parallel mode is supported
 def test_multiheadattention(model, query_data, key_value_data):
-    config = hls4ml.utils.config_from_keras_model(model, granularity='name', backend='Vivado')
-    output_dir = str(test_root_path / 'hls4mlprj_multiheadattention_Vivado_io_parallel')
+    config = hls4ml.utils.config_from_keras_model(model, granularity='name', backend='Vitis')
+    output_dir = str(test_root_path / 'hls4mlprj_multiheadattention_Vitis_io_parallel')
     hls_model = hls4ml.converters.convert_from_keras_model(
-        model, backend='Vivado', hls_config=config, io_type='io_parallel', output_dir=output_dir
+        model, backend='Vitis', hls_config=config, io_type='io_parallel', output_dir=output_dir
     )
     hls_model.compile()
 

test/pytest/test_multiheadattention_pytorch.py

@@ -37,7 +37,7 @@ def forward(self, query, key, value):
         return output
 
 
-# Currently only Vivado in io_parallel mode is supported
+# Currently only Vitis in io_parallel mode is supported
 def test_multiheadattention(query_data, key_value_data):
     model = MultiHeadAttentionModel()
     model.eval()
@@ -46,13 +46,13 @@ def test_multiheadattention(query_data, key_value_data):
         model,
         [(seq_len, embed_dim), (seq_len, embed_dim), (seq_len, embed_dim)],
         granularity='name',
-        backend='Vivado',
+        backend='Vitis',
         channels_last_conversion='off',
         transpose_outputs=False,
     )
-    output_dir = str(test_root_path / 'hls4mlprj_multiheadattention_pytorch_Vivado_io_parallel')
+    output_dir = str(test_root_path / 'hls4mlprj_multiheadattention_pytorch_Vitis_io_parallel')
     hls_model = hls4ml.converters.convert_from_pytorch_model(
-        model, backend='Vivado', hls_config=config, io_type='io_parallel', output_dir=output_dir
+        model, backend='Vitis', hls_config=config, io_type='io_parallel', output_dir=output_dir
     )
     hls_model.compile()