Added caching of the metadata to improve the Z2 overlay load time (#1391

)

pynq/pl.py

@@ -65,7 +65,12 @@ def timestamp(cls):
             Bitstream download timestamp.
 
         """
-        return Device.active_device.timestamp
+        if hasattr(Device.active_device, "timestamp"):
+            return Device.active_device.timestamp
+
+        if global_state_file_exists():
+            gs = load_global_state()
+            return gs.timestamp
 
     @property
     def dict_views(cls):

pynq/pl_server/device.py

@@ -92,6 +92,7 @@ def clear_state(dict_in):
             dict_in[k] = None
     return dict_in
 
+
 class Device(metaclass=DeviceMeta):
     """Construct a new Device Instance
 
@@ -114,6 +115,7 @@ def __init__(self, tag, warn=False):
     def set_bitfile_name(self, bitfile_name: str) -> None:
         self.bitfile_name = bitfile_name
         self.parser = self.get_bitfile_metadata(self.bitfile_name)
+        self.mem_dict = self.parser.mem_dict
         self.ip_dict = self.parser.ip_dict
         self.gpio_dict = self.parser.gpio_dict
         self.interrupt_pins = self.parser.interrupt_pins
@@ -418,25 +420,27 @@ def post_download(self, bitstream, parser, name: str = "Unknown"):
             )
             self.reset(parser, bitstream.timestamp, bitstream.bitfile_name)
 
-            from .global_state import GlobalState, save_global_state
-
-            gs = GlobalState(bitfile_name=bitstream.bitfile_name,
-                             active_name=name,
-                             psddr=self.mem_dict.get("PSDDR", {}))
-            ip = self.ip_dict
-            for sd_name, details in ip.items():
-                if details["type"] in [
-                    "xilinx.com:ip:pr_axi_shutdown_manager:1.0",
-                    "xilinx.com:ip:dfx_axi_shutdown_manager:1.0",
-                ]:
-                    gs.add(name=sd_name, addr=details["phys_addr"])
-            save_global_state(gs)
-
-            if hasattr(self, "systemgraph"):
-                if not self.systemgraph is None:
-                    import os
-                    STATE_DIR = os.path.dirname(__file__)
-                    self.systemgraph.export(path=f"{STATE_DIR}/_current_metadata.json")
+            if not hasattr(parser, "_from_cache"):
+                from .global_state import GlobalState, save_global_state
+
+                gs = GlobalState(bitfile_name=bitstream.bitfile_name,
+                                 timestamp=bitstream.timestamp,
+                                 active_name=name,
+                                 psddr=self.mem_dict.get("PSDDR", {}))
+                ip = self.ip_dict
+                for sd_name, details in ip.items():
+                    if details["type"] in [
+                        "xilinx.com:ip:pr_axi_shutdown_manager:1.0",
+                        "xilinx.com:ip:dfx_axi_shutdown_manager:1.0",
+                    ]:
+                        gs.add(name=sd_name, addr=details["phys_addr"])
+                save_global_state(gs)
+
+                if hasattr(self, "systemgraph"):
+                    if not self.systemgraph is None:
+                        import os
+                        STATE_DIR = os.path.dirname(__file__)
+                        self.systemgraph.export(path=f"{STATE_DIR}/_current_metadata.json")
 
     def has_capability(self, cap):
         """Test if the device as a desired capability

pynq/pl_server/embedded_device.py

@@ -178,6 +178,16 @@ def is_xsa(self):
             return True
         return False
 
+    def _cache_exists(self)->bool:
+        """ Checks to see if this bitstream is already on the system and
+        use the cached metadata """
+        from .global_state import bitstream_hash, load_global_state, global_state_file_exists
+        if global_state_file_exists():
+            glob_state = load_global_state()
+            return glob_state.bitfile_hash == bitstream_hash(self._filepath)
+        return False
+
+
     def get_parser(self, partial:bool=False):
         """Returns a parser object for the bitstream
 
@@ -199,9 +209,16 @@ def get_parser(self, partial:bool=False):
             if partial:
                 parser = HWH(hwh_data=hwh_data)
             else:
-                parser = RuntimeMetadataParser(
-                    Metadata(input=self._filepath.with_suffix(".hwh"))
-                )
+                if self._cache_exists():
+                    metadata_state_file = Path(f"{os.path.dirname(__file__)}/_current_metadata.json")
+                    if os.path.isfile(metadata_state_file):
+                        parser = RuntimeMetadataParser(Metadata(input=metadata_state_file))
+                        parser._from_cache = True
+                    else:
+                        parser = RuntimeMetadataParser(Metadata(input=self._filepath.with_suffix(".hwh")))
+                else:
+                    parser = RuntimeMetadataParser(Metadata(input=self._filepath.with_suffix(".hwh")))
+
 
             if xclbin_data is None:
                 xclbin_data = _create_xclbin(parser.mem_dict)
@@ -583,28 +600,30 @@ def set_axi_port_width(self, parser):
                         Register(addr)[f[0] : f[1]] = ZU_AXIFM_VALUE[width]
 
     def download(self, bitstream, parser=None):
-        if parser is None:
-            from .xclbin_parser import XclBin
 
-            parser = XclBin(xclbin_data=DEFAULT_XCLBIN)
+        if not hasattr(parser, "_from_cache"):
+            if parser is None:
+                from .xclbin_parser import XclBin
 
-        if not bitstream.binfile_name:
-            _preload_binfile(bitstream, parser)
+                parser = XclBin(xclbin_data=DEFAULT_XCLBIN)
 
-        if not bitstream.partial:
-            self.shutdown()
-            flag = 0
-        else:
-            flag = 1
+            if not bitstream.binfile_name:
+                _preload_binfile(bitstream, parser)
+
+            if not bitstream.partial:
+                self.shutdown()
+                flag = 0
+            else:
+                flag = 1
 
-        with open(self.BS_FPGA_MAN_FLAGS, "w") as fd:
-            fd.write(str(flag))
-        with open(self.BS_FPGA_MAN, "w") as fd:
-            fd.write(bitstream.binfile_name)
+            with open(self.BS_FPGA_MAN_FLAGS, "w") as fd:
+                fd.write(str(flag))
+            with open(self.BS_FPGA_MAN, "w") as fd:
+                fd.write(bitstream.binfile_name)
 
-        self.set_axi_port_width(parser)
+            self.set_axi_port_width(parser)
 
-        self._xrt_download(parser.xclbin_data)
+            self._xrt_download(parser.xclbin_data)
         super().post_download(bitstream, parser, self.name)
 
     def get_bitfile_metadata(self, bitfile_name:str, partial:bool=False):

pynq/pl_server/global_state.py

@@ -15,13 +15,25 @@ class ShutdownIP(BaseModel):
     name: str
     base_addr: int
 
+import hashlib
+def bitstream_hash(filename:str)->int:
+    """ Returns a hash of the bitstream """
+    h = hashlib.sha1()
+    with open(filename, 'rb') as file:
+        chunk=0
+        while chunk != b'':
+            chunk = file.read(1024)
+            h.update(chunk)
+    return h.hexdigest()
 
 class GlobalState(BaseModel):
     """A class that is used to globally keep track on some details of the currently
     configured bitstream"""
 
     bitfile_name: str
     active_name: str
+    timestamp : str
+    bitfile_hash : str = ""
     shutdown_ips: Dict[str, ShutdownIP] = {}
     psddr: Dict = {}
 
@@ -32,6 +44,9 @@ def add(self, name: str, addr: int) -> None:
         if name not in self.shutdown_ips:
             self.shutdown_ips[name] = ShutdownIP(name=name, base_addr=addr)
 
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.bitfile_hash = bitstream_hash(self.bitfile_name)
 
 def initial_global_state_file_boot_check()->None:
     """ Performs a check to see if this is a coldstart, if it is then clear the