fpga: lib: Update xport_sv

- Detect dropped words at the dispatch level. This prevents
   an overflow on CHDR from block CPU.
 - Dropped packets are recorded as CPU or CHDR drop count
 - Refactor to put chdr_xport_adapter.sv in different clock
   domain to improve timing
 - Unwrinkle tkeep/trailing transitions


Original-commit: 7f86724ec3387f75d39d683b2ac5f5152e714c74

lib/rfnoc/xport_sv/chdr_xport_adapter.sv

@@ -511,15 +511,29 @@ module chdr_xport_adapter #(
     data_fifo_o.tready  = au.tready & pass_packet;
   end
 
+  // Clock Crossing to the ethernet clock domain
+  logic [47:0] e_my_mac;
+  logic [31:0] e_my_ip;
+  logic [15:0] e_my_udp_chdr_port;
+  // crossing clock boundaries.
+  // my_mac, my_ip, my_udp_chdr_port must be written
+  // prior to traffic, or an inconsistent version will
+  // exist for a clock period or 2.  This would be better
+  // done with a full handshake.
+  synchronizer #(.WIDTH(96),.STAGES(1))
+    e_info_sync (.clk(eth_rx.clk),.rst(eth_rx.rst),
+                 .in({my_mac,my_ip,my_udp_chdr_port}),
+                 .out({e_my_mac,e_my_ip,e_my_udp_chdr_port}));
+
   // add the UDP header back on before sending to EthTx
   eth_ipv4_add_udp #(
     .PREAMBLE_BYTES(PREAMBLE_BYTES),
     .MAX_PACKET_BYTES(MAX_PACKET_BYTES)
   ) add_udp_i (
     .i(au), .o(eth_tx),
-    .mac_src(my_mac),
+    .mac_src(e_my_mac),
-    .ip_src(my_ip),
+    .ip_src(e_my_ip),
-    .udp_src(my_udp_chdr_port),
+    .udp_src(e_my_udp_chdr_port),
     .mac_dst(au_mac_dst),
     .ip_dst(au_ip_dst),
     .udp_dst(au_udp_dst)

lib/rfnoc/xport_sv/eth_interface_tb/eth_ifc_tb.sv

@@ -48,7 +48,7 @@ module eth_ifc_tb #(
   // Include for register offsets
   `include "../eth_regs.vh"
   // allows the DUT to push full words and tb does not check tuser/tkeep of packets it's transmitting
-  localparam IGNORE_EXTRA_DATA = 1;
+  localparam IGNORE_EXTRA_DATA = 0;
 
   //---------------------------------------------------------------------------
   // Clocks
@@ -68,10 +68,10 @@ module eth_ifc_tb #(
   TestExec test = new();
 
   localparam MAX_PACKET_BYTES = 2**16;
-  AxiStreamIf #(.DATA_WIDTH(ENET_W),.USER_WIDTH(ENET_USER_W),.TKEEP(0),
+  AxiStreamIf #(.DATA_WIDTH(ENET_W),.USER_WIDTH(ENET_USER_W),
                 .MAX_PACKET_BYTES(MAX_PACKET_BYTES))
     eth_tx (eth_clk, eth_reset);
-  AxiStreamIf #(.DATA_WIDTH(ENET_W),.USER_WIDTH(ENET_USER_W),.TKEEP(0),
+  AxiStreamIf #(.DATA_WIDTH(ENET_W),.USER_WIDTH(ENET_USER_W),
                 .MAX_PACKET_BYTES(MAX_PACKET_BYTES))
     eth_rx (eth_clk, eth_reset);
 
@@ -86,7 +86,7 @@ module eth_ifc_tb #(
     e2c    (clk, reset);
 
   // Bus functional model for a axi_stream controller
-  AxiStreamBfm #(.DATA_WIDTH(ENET_W),.USER_WIDTH(ENET_USER_W),.TKEEP(0),
+  AxiStreamBfm #(.DATA_WIDTH(ENET_W),.USER_WIDTH(ENET_USER_W),
                  .MAX_PACKET_BYTES(MAX_PACKET_BYTES)) eth =
     new(.master(eth_rx), .slave(eth_tx));
   AxiStreamBfm #(.DATA_WIDTH(CHDR_W),.USER_WIDTH(CHDR_USER_W),.TKEEP(0),.TUSER(0)) v =
@@ -123,8 +123,12 @@ module eth_ifc_tb #(
 
     eth_ipv4_interface #(
      .PREAMBLE_BYTES(PREAMBLE_BYTES),
-     .PROTOVER(PROTOVER), .MTU(MTU), .NODE_INST(NODE_INST),
+     .CPU_FIFO_SIZE(MTU),
-     .REG_AWIDTH(REG_AWIDTH), .RT_TBL_SIZE(RT_TBL_SIZE),
+     .CHDR_FIFO_SIZE(MTU),
+     .PROTOVER(PROTOVER),
+     .NODE_INST(NODE_INST),
+     .REG_AWIDTH(REG_AWIDTH),
+     .RT_TBL_SIZE(RT_TBL_SIZE),
      .BASE(BASE),.SYNC(SYNC),
      .ENET_W(ENET_W),.CPU_W(CPU_W),.CHDR_W(CHDR_W)
     ) eth_interface (
@@ -168,6 +172,7 @@ module eth_ifc_tb #(
     always_comb begin
      eth_tx.tdata  = eth_tx_tdata;
      eth_tx.tuser  = eth_tx_tuser;
+     eth_tx.tkeep  = eth_tx.trailing2keep(eth_tx_tuser);
      eth_tx.tlast  = eth_tx_tlast;
      eth_tx.tvalid = eth_tx_tvalid;
      eth_tx_tready = eth_tx.tready;
@@ -202,7 +207,7 @@ module eth_ifc_tb #(
     end
 
     eth_interface #(
-     .PROTOVER(PROTOVER), .MTU(MTU), .NODE_INST(NODE_INST),
+     .PROTOVER(PROTOVER), .NODE_INST(NODE_INST), .MTU(MTU),
      .REG_AWIDTH(REG_AWIDTH), .RT_TBL_SIZE(RT_TBL_SIZE),
      .BASE(BASE)
     ) eth_interface (
@@ -210,9 +215,6 @@ module eth_ifc_tb #(
      .my_udp_port   (my_udp_chdr_port)
     );
   end
-  always_comb begin
-    eth_tx.tkeep = '1;
-  end
 
   task automatic reg_wr (
     // Register port: Write port (domain: clk)
@@ -310,7 +312,10 @@ module eth_ifc_tb #(
     reg_rd_check(REG_BRIDGE_MAC_MSB,DEF_BRIDGE_MAC_ADDR[47:32]);
     reg_rd_check(REG_BRIDGE_IP,DEF_BRIDGE_IP_ADDR);
     reg_rd_check(REG_BRIDGE_UDP,DEF_BRIDGE_UDP_PORT);
-
+    if (SV_ETH_IFC) begin
+      reg_rd_check(REG_CHDR_DROPPED,0);
+      reg_rd_check(REG_CPU_DROPPED,0);
+    end
     test.end_test();
   endtask : test_registers
 
@@ -330,7 +335,7 @@ module eth_ifc_tb #(
     typedef AxiStreamPacket   #(CHDR_W,CHDR_USER_W) ChdrAxisPacket_t;
     typedef ChdrPacket        #(CHDR_W,CHDR_USER_W) ChdrPacket_t;
 
-  task automatic test_ethcpu(int num_samples[$], int ERROR_PROB=2);
+  task automatic test_ethcpu(int num_samples[$], int ERROR_PROB=2, int EXPECT_DROPS=0);
     TestExec test_e2c = new();
     automatic EthXportPacket_t send[$];
     automatic CpuXportPacket_t expected[$];
@@ -402,20 +407,41 @@ module eth_ifc_tb #(
         end
       end
       begin //rx_thread
-        foreach(expected[i]) begin
+        if (EXPECT_DROPS > 0) begin
-          automatic CpuAxisPacket_t  actual_a;
+          automatic int pkt_num = 0;
-          automatic CpuXportPacket_t actual = new();
+          automatic int drop_count = 0;
-          cpu.get(actual_a);
+          while (expected.size() > 0) begin
-          actual.import_axis(actual_a);
+            automatic CpuAxisPacket_t  actual_a;
-          actual.tuser_to_tkeep();
+            automatic CpuXportPacket_t actual = new();
-//          $display({"****************::%s::**********************\n",
+            cpu.get(actual_a);
-//                    "**send** \n%s\n**expected**\n%s\n**actual**\n%s \n".
+            actual.import_axis(actual_a);
-//                    "%d byte with error %d"},
+            actual.tuser_to_tkeep();
-//                   TEST_NAME,
+            while (expected.size > 0 && actual.compare_no_user(expected[0],.PRINT_LVL(0))) begin
-//                   send[i].sprint(),expected[i].sprint(),actual.sprint(),
+              void'(expected.pop_front());
-//                   num_samples[i],send[i].has_error());
+              ++drop_count;
-
+              ++pkt_num;
-          `ASSERT_ERROR(!actual.compare_w_sof(expected[i]),"failed to send packet to e2c");
+               $display("Droped packet %d",pkt_num);
+              `ASSERT_ERROR(drop_count < EXPECT_DROPS,"Exceeded anticipated number of dropped packets e2c");
+            end
+            if (expected.size() > 0) begin
+              ++pkt_num;
+              $display("Rcvd packet   %d",pkt_num);
+              void'(expected.pop_front());
+            end
+          end
+          if (SV_ETH_IFC) begin
+            $display("Verify drop count is %d",drop_count);
+            reg_rd_check(REG_CPU_DROPPED,drop_count);
+          end
+        end else begin
+          foreach(expected[i]) begin
+            automatic CpuAxisPacket_t  actual_a;
+            automatic CpuXportPacket_t actual = new();
+            cpu.get(actual_a);
+            actual.import_axis(actual_a);
+            actual.tuser_to_tkeep();
+            `ASSERT_ERROR(!actual.compare_w_sof(expected[i]),"failed to send packet to e2c");
+          end
         end
       end
     join
@@ -512,7 +538,9 @@ module eth_ifc_tb #(
           wait_for_udp_packets(DEF_DEST_UDP_PORT);
           eth.get(actual_a);
           actual.import_axis(actual_a);
-          actual.tuser_to_tkeep();
+          if (!SV_ETH_IFC) begin
+            actual.tuser_to_tkeep();
+          end
          `ASSERT_ERROR(!actual.compare_w_pad(expected[i],!SV_ETH_IFC),"failed to send packet to c2e");
         end
       end
@@ -556,7 +584,7 @@ module eth_ifc_tb #(
     return chdr_pkt;
   endfunction : unflatten_chdr
 
-  task automatic test_ethchdr(int num_samples[$], int ERROR_PROB=2);
+  task automatic test_ethchdr(int num_samples[$], int ERROR_PROB=2, int EXPECT_DROPS=0);
     TestExec test_e2v = new();
     automatic EthXportPacket_t send[$];
     automatic ChdrXportPacket_t expected[$];
@@ -655,13 +683,41 @@ module eth_ifc_tb #(
         end
       end
       begin //rx_thread
-        foreach(expected[i]) begin
+        if (EXPECT_DROPS > 0) begin
-          automatic ChdrAxisPacket_t  actual_a;
+          automatic int pkt_num = 0;
-          automatic ChdrXportPacket_t actual = new();
+          automatic int drop_count = 0;
-          v.get(actual_a);
+          while (expected.size() > 0) begin
-          actual.import_axis(actual_a);
+            automatic ChdrAxisPacket_t  actual_a;
-          actual.tuser_to_tkeep();
+            automatic ChdrXportPacket_t actual = new();
-         `ASSERT_ERROR(!actual.compare_no_user(expected[i]),"failed to send packet e2v");
+            v.get(actual_a);
+            actual.import_axis(actual_a);
+            actual.tuser_to_tkeep();
+            while (expected.size > 0 && actual.compare_no_user(expected[0],.PRINT_LVL(0))) begin
+              void'(expected.pop_front());
+              ++drop_count;
+              ++pkt_num;
+               $display("Droped packet %d",pkt_num);
+              `ASSERT_ERROR(drop_count < EXPECT_DROPS,"Exceeded anticipated number of dropped packets e2v");
+            end
+            if (expected.size() > 0) begin
+              ++pkt_num;
+              $display("Rcvd packet   %d",pkt_num);
+              void'(expected.pop_front());
+            end
+          end
+          if (SV_ETH_IFC) begin
+            $display("Verify drop count is %d",drop_count);
+            reg_rd_check(REG_CHDR_DROPPED,drop_count);
+          end
+        end else begin
+          foreach(expected[i]) begin
+            automatic ChdrAxisPacket_t  actual_a;
+            automatic ChdrXportPacket_t actual = new();
+            v.get(actual_a);
+            actual.import_axis(actual_a);
+            actual.tuser_to_tkeep();
+           `ASSERT_ERROR(!actual.compare_no_user(expected[i]),"failed to send packet e2v");
+          end
         end
       end
     join
@@ -785,9 +841,8 @@ module eth_ifc_tb #(
           wait_for_udp_packets(.udp_dest_port(0));
           eth.get(actual_a);
           actual.import_axis(actual_a);
-          actual.tuser_to_tkeep();
           actual_raw = actual.dump_bytes();
-          repeat(PREAMBLE_BYTES) actual_raw.pop_front();
+          repeat(PREAMBLE_BYTES) void'(actual_raw.pop_front());
           decode_udp_pkt(actual_raw,eth_hdr,ipv4_hdr,udp_hdr,chdr_raw);
           chdr_pkt = unflatten_chdr(chdr_raw);
           // fills remainder of packet with zeros
@@ -939,9 +994,8 @@ module eth_ifc_tb #(
 
           eth.get(actual_a);
           actual.import_axis(actual_a);
-          actual.tuser_to_tkeep();
           actual_raw = actual.dump_bytes();
-          repeat(PREAMBLE_BYTES) actual_raw.pop_front();
+          repeat(PREAMBLE_BYTES) void'(actual_raw.pop_front());
           decode_udp_pkt(actual_raw,eth_hdr,ipv4_hdr,udp_hdr,chdr_raw);
           chdr_pkt = unflatten_chdr(chdr_raw);
           // fills remainder of packet with zeros
@@ -966,6 +1020,7 @@ module eth_ifc_tb #(
   initial begin : tb_main
    automatic int num_samples[$];
    automatic int cpu_num_samples[$];
+   automatic int expected_drops;
    localparam QUICK = 0;
    test.start_test({TEST_NAME,"Wait for Reset"}, 10us);
    clk_gen.reset();
@@ -981,6 +1036,42 @@ module eth_ifc_tb #(
 
    test_chdr_endpoint();
 
+   // Check what happens if the input bandwidth exceeds
+   // the devices ability to consume packets
+   // This can happen in matched bandwidth cases
+   // if there is hold off from upstream
+   // Dropped packets exceed the drop count cause an error
+   // The actual droped count is compared versus the real count
+   test.start_test({TEST_NAME,"::Input overrun"}, 200us);
+
+   eth.set_master_stall_prob(0);
+   eth.set_slave_stall_prob(0);
+   cpu.set_master_stall_prob(0);
+   cpu.set_slave_stall_prob(0);
+   v.set_master_stall_prob(0);
+   v.set_slave_stall_prob(0);
+
+   num_samples = {7936,7936,7936,7936,7936,320,
+                  7936,7936,7936,7936,7936,320};
+
+   // The actual number of expected drops depends on the
+   // bus width difference between ENET_W and CHDR/CPU_W
+
+   // in this SIM unlimited etherent bandwidth is coming in at over 300 MHZ
+   // and output runs at 200 MHZ.  This causes excess BW on transmitter even when matched.
+   expected_drops = 9;
+   
+   test_ethchdr(num_samples,.EXPECT_DROPS(expected_drops),.ERROR_PROB(0));
+   test_ethcpu(num_samples,.EXPECT_DROPS(expected_drops),.ERROR_PROB(0));
+   test.end_test();
+
+   eth.set_master_stall_prob(38);
+   eth.set_slave_stall_prob(38);
+   cpu.set_master_stall_prob(38);
+   cpu.set_slave_stall_prob(38);
+   v.set_master_stall_prob(38);
+   v.set_slave_stall_prob(38);
+
    num_samples = {1,2,3,4,5,6,7,8,
                   ENET_W/8-1,ENET_W/8,ENET_W/8+1,
                   2*ENET_W/8-1,2*ENET_W/8,2*ENET_W/8+1,

lib/rfnoc/xport_sv/eth_ipv4_chdr_adapter.sv

@@ -16,15 +16,15 @@
 //
 // Parameters:
 //   - PROTOVER: RFNoC protocol version {8'd<major>, 8'd<minor>}
-//   - MTU: Log2 of the MTU of the packet in 64-bit words
+//   - CPU_FIFO_SIZE: Log2 of the FIFO depth (in bytes) for the CPU egress path
-//   - CPU_FIFO_SIZE: Log2 of the FIFO depth (in 64-bit words) for the CPU egress path
+//   - CHDR_FIFO_SIZE: Log2 of the FIFO depth (in bytes) for the CHDR egress path
 //   - RT_TBL_SIZE: Log2 of the depth of the return-address routing table
 //   - NODE_INST: The node type to return for a node-info discovery
 //   - DROP_UNKNOWN_MAC: Drop packets not addressed to us?
 //   - DROP_MIN_PACKET: Drop packets smaller than 64 bytes?
 //   - PREAMBLE_BYTES: Number of bytes of Preamble expected
-//   - ADD_SOF: Add a SOF indication into the tuser field
+//   - ADD_SOF: Add a SOF indication into the tuser field of the e2c path.
-//              If false use TKEEP instead of USER
+//              If false use TKEEP instead of USER.
 //   - SYNC: Set if MAC is not the same as bus_clk
 //   - ENET_W: Width of the link to the Ethernet MAC
 //   - CPU_W: Width of the CPU interface
@@ -45,8 +45,8 @@
 
 module eth_ipv4_chdr_adapter #(
   logic [15:0] PROTOVER         = {8'd1, 8'd0},
-  int          MTU              = 10,
+  int          CPU_FIFO_SIZE    = $clog2(8*1024),
-  int          CPU_FIFO_SIZE    = MTU,
+  int          CHDR_FIFO_SIZE   = $clog2(8*1024),
   int          RT_TBL_SIZE      = 6,
   int          NODE_INST        = 0,
   bit          DROP_UNKNOWN_MAC = 0,
@@ -58,6 +58,7 @@ module eth_ipv4_chdr_adapter #(
   int          CPU_W            = 64,
   int          CHDR_W           = 64
 )(
+
   // Device info
   input  logic [15:0] device_id,
   // Device addresses
@@ -65,6 +66,9 @@ module eth_ipv4_chdr_adapter #(
   input  logic [31:0] my_ip,
   input  logic [15:0] my_udp_chdr_port,
 
+  output logic        chdr_dropped,
+  output logic        cpu_dropped,
+
   // Ethernet MAC
   AxiStreamIf.master eth_tx, // tUser = {1'b0,trailing bytes};
   AxiStreamIf.slave  eth_rx, // tUser = {error,trailing bytes};
@@ -90,6 +94,10 @@ module eth_ipv4_chdr_adapter #(
 
   `include "eth_constants.vh"
 
+
+  //   tUser = {error,trailing_bytes}
+  AxiStreamIf #(.DATA_WIDTH(ENET_W),.USER_WIDTH(ENET_USER_W),.TKEEP(0)) eth_rx1(eth_rx.clk,eth_rx.rst);
+
   //---------------------------------------
   // E2V and E2C DEMUX
   //---------------------------------------
@@ -98,9 +106,8 @@ module eth_ipv4_chdr_adapter #(
   //   tUser = {*not used*}
   AxiStreamIf #(.DATA_WIDTH(CHDR_W),.TKEEP(0),.TUSER(0)) e2v2(e2v.clk,e2v.rst);
   //   tUser = {*not used*}
-  AxiStreamIf #(.DATA_WIDTH(CHDR_W),.TKEEP(0),.TUSER(0)) e2v4(e2v.clk,e2v.rst);
+  AxiStreamIf #(.DATA_WIDTH(CHDR_W),.TKEEP(0),.TUSER(0)) e2v3(e2v.clk,e2v.rst);
-  //   tUser = {*not used*}
+
-  AxiStreamIf #(.DATA_WIDTH(CHDR_W),.TKEEP(0),.TUSER(0)) e2v5(e2v.clk,e2v.rst);
 
   //   tUser = {1'b0,trailing bytes}
   AxiStreamIf #(.DATA_WIDTH(ENET_W),.USER_WIDTH(ENET_USER_W),.TKEEP(0)) e2c1(eth_rx.clk,eth_rx.rst);
@@ -109,35 +116,35 @@ module eth_ipv4_chdr_adapter #(
                 .TKEEP(!ADD_SOF), .TUSER(ADD_SOF))
      e2c2(e2c.clk,e2c.rst);
 
-  logic [47:0] e_my_mac;
+  // The older implementation connected with tUser containing trailing bytes
-  logic [31:0] e_my_ip;
+  // The newer implementation brings in TKEEP
-  logic [15:0] e_my_udp_chdr_port;
+  // inside this block we expect trailing bytes.
-  // crossing clock boundaries. 
+  always_comb begin
-  // my_mac, my_ip,,my_udp_chdr_port must be written 
+    `AXI4S_ASSIGN(eth_rx1,eth_rx)
-  // prior to traffic, or an inconsistent version will
+    if (eth_rx.TKEEP) begin
-  // exist for a clock period or 2.  This would be better
+      eth_rx1.tuser = {eth_rx.tuser[ENET_USER_W-1],eth_rx.keep2trailing(eth_rx.tkeep)};
-  // done with a full handshake.
+    end
-  synchronizer #(.WIDTH(96),.STAGES(1))
+  end
-    e_info_sync (.clk(eth_rx.clk),.rst(eth_rx.rst),
-                 .in({my_mac,my_ip,my_udp_chdr_port}),
-                 .out({e_my_mac,e_my_ip,e_my_udp_chdr_port}));
 
   // Ethernet sink. Inspects packet and dispatches
   // to the correct port.
   eth_ipv4_chdr_dispatch #(
     .CPU_FIFO_SIZE(CPU_FIFO_SIZE),
+    .CHDR_FIFO_SIZE(CHDR_FIFO_SIZE),
     .PREAMBLE_BYTES(PREAMBLE_BYTES),
     .MAX_PACKET_BYTES(MAX_PACKET_BYTES),
     .DROP_UNKNOWN_MAC(DROP_UNKNOWN_MAC),
     .DROP_MIN_PACKET(DROP_MIN_PACKET),
     .ENET_W(ENET_W)
   ) eth_dispatch_i (
-    .eth_rx           (eth_rx),
+    .eth_rx           (eth_rx1),
     .e2v              (e2v1),
     .e2c              (e2c1),
-    .my_mac           (e_my_mac),
+    .my_mac           (my_mac),
-    .my_ip            (e_my_ip),
+    .my_ip            (my_ip),
-    .my_udp_chdr_port (e_my_udp_chdr_port)
+    .my_udp_chdr_port (my_udp_chdr_port),
+    .chdr_dropped     (chdr_dropped),
+    .cpu_dropped      (cpu_dropped)
   );
 
   //---------------------------------------
@@ -222,7 +229,7 @@ module eth_ipv4_chdr_adapter #(
     .my_udp_chdr_port    (my_udp_chdr_port),
 
     .eth_rx     (e2v2), // from ethernet
-    .e2v        (e2v4), // to   CHDR
+    .e2v        (e2v3), // to   CHDR
     // optional loop from ethernet to ethernet to talk to node
     .v2e        (v2e),  // from CHDR
     .eth_tx     (v2e1D)  // to   ethernet
@@ -261,28 +268,14 @@ module eth_ipv4_chdr_adapter #(
   // E2V Output Buffering
   //---------------------------------------
 
-  // The transport should hook up to a crossbar downstream, which
-  // may backpressure this module because it is in the middle of
-  // transferring a packet. To ensure that upstream logic is not
-  // blocked, we instantiate one packet worth of buffering here.
-  axi4s_fifo #(
-    .SIZE(MTU)
-  ) chdr_fifo_i (
-    .clear(1'b0),.space(),.occupied(),
-    .i(e2v4),.o(e2v5)
-  );
-
   if (DEBUG) begin
-    `AXI4S_DEBUG_ASSIGN(e2v,e2v5)
+    `AXI4S_DEBUG_ASSIGN(e2v,e2v3)
   end else begin
     always_comb begin : e2v_direct_assign
-      `AXI4S_ASSIGN(e2v,e2v5)
+      `AXI4S_ASSIGN(e2v,e2v3)
     end
   end
 
-
-
-
   //---------------------------------------
   // C2E Path
   //---------------------------------------
@@ -291,7 +284,7 @@ module eth_ipv4_chdr_adapter #(
                 .TKEEP(c2e.TKEEP),.TUSER(c2e.TUSER))
     c2eD(c2e.clk,c2e.rst);
   //   tUser = {1'b0,trailing bytes}
-  AxiStreamIf #(.DATA_WIDTH(ENET_W),.USER_WIDTH(ENET_USER_W),.TKEEP(0)) c2e1(eth_rx.clk,eth_rx.rst);
+  AxiStreamIf #(.DATA_WIDTH(ENET_W),.USER_WIDTH(ENET_USER_W),.TKEEP(0),.MAX_PACKET_BYTES(MAX_PACKET_BYTES)) c2e1(eth_rx.clk,eth_rx.rst);
   //   tUser = {1'b0,trailing bytes}
   AxiStreamIf #(.DATA_WIDTH(ENET_W),.USER_WIDTH(ENET_USER_W),.TKEEP(0),.MAX_PACKET_BYTES(MAX_PACKET_BYTES)) c2e2(eth_rx.clk,eth_rx.rst);
   //   tUser = {1'b0,trailing bytes}
@@ -332,8 +325,8 @@ module eth_ipv4_chdr_adapter #(
     // Add pad of PREAMBLE_BYTES empty bytes to the ethernet packet going
     // from the CPU to the SFP. This padding added before MAC addresses
     // aligns the source and destination IP addresses, UDP headers etc.
-    // Note that the xge_mac_wrapper strips this padding to recreate the ethernet
+    // Note that the xge_mac_wrapper strips this padding to recreate the
-    // packet
+    // ethernet packet.
     axi4s_add_bytes #(.ADD_START(0),.ADD_BYTES(PREAMBLE_BYTES)
     ) add_header (
      .i(c2e1), .o(c2e2)
@@ -380,7 +373,7 @@ module eth_ipv4_chdr_adapter #(
 
     always_comb begin : c2e3_pad
       if (pad_state == ST_IDLE) begin
-        //force to a full word
+        // force to a full word
         // preserve SOF if it's there, but force
         // trailing bytes to zero (full word)
         c2e3.tuser = 0;
@@ -429,12 +422,13 @@ module eth_ipv4_chdr_adapter #(
   //---------------------------------------
   // V2E and C2E MUX
   //---------------------------------------
+  //   tUser = {1'b0,trailing bytes}
+  AxiStreamIf #(.DATA_WIDTH(ENET_W),.USER_WIDTH(ENET_USER_W),.TKEEP(0)) eth_tx1 (eth_rx.clk,eth_rx.rst);
+  //   tUser = {1'b0,trailing bytes}
+  AxiStreamIf #(.DATA_WIDTH(ENET_W),.USER_WIDTH(ENET_USER_W)) eth_tx2 (eth_rx.clk,eth_rx.rst);
+
+
   logic c2e3_tready;
-  logic eth_tx1_tlast;
-  logic eth_tx1_tvalid;
-  logic eth_tx1_tready;
-  logic [ENET_W-1:0]      eth_tx1_tdata;
-  logic [ENET_USER_W-1:0] eth_tx1_tuser;
   always_comb begin
     c2e3.tready    = c2e3_tready;
   end
@@ -444,30 +438,47 @@ module eth_ipv4_chdr_adapter #(
     .clk(eth_rx.clk), .reset(eth_rx.rst), .clear(1'b0),
     .i_tdata({c2e3.tuser, c2e3.tdata, v2e3.tuser, v2e3.tdata}), .i_tlast({c2e3.tlast, v2e3.tlast}),
     .i_tvalid({c2e3.tvalid, v2e3.tvalid}), .i_tready({c2e3_tready, v2e3.tready}),
-    .o_tdata({eth_tx1_tuser, eth_tx1_tdata}), .o_tlast(eth_tx1_tlast),
+    .o_tdata({eth_tx1.tuser, eth_tx1.tdata}), .o_tlast(eth_tx1.tlast),
-    .o_tvalid(eth_tx1_tvalid), .o_tready(eth_tx1_tready)
+    .o_tvalid(eth_tx1.tvalid), .o_tready(eth_tx1.tready)
   );
 
+
+
   // Clean up the noisy mux output.  I suspect it is annoying
-  // the xilinx cores that tlast and tuser(tkeep) flop around
+  // the Xilinx cores that tlast and tuser(tkeep) flop around
   // when tvalid isn't true.
   always_comb begin : eth_tx_assign
-    if (eth_tx1_tvalid) begin
+    if (eth_tx1.tvalid) begin
-      eth_tx.tvalid = 1'b1;
+      eth_tx2.tvalid = 1'b1;
-      eth_tx.tdata  = eth_tx1_tdata;
+      eth_tx2.tdata  = eth_tx1.tdata;
-      eth_tx.tlast  = eth_tx1_tlast;
+      eth_tx2.tlast  = eth_tx1.tlast;
-      if (eth_tx1_tlast) begin
+      // driving both tuser and tkeep
-        eth_tx.tuser  = eth_tx1_tuser;
+      if (eth_tx1.tlast) begin
+        eth_tx2.tuser  = eth_tx1.tuser;
+        eth_tx2.tkeep  = eth_tx1.trailing2keep(eth_tx1.tuser);
       end else begin
-        eth_tx.tuser  = '0;
+        eth_tx2.tuser  = '0;
+        eth_tx2.tkeep  = '1;
       end
     end else begin
-      eth_tx.tvalid = 1'b0;
+      eth_tx2.tvalid = 1'b0;
-      eth_tx.tdata  = 'X; // use X so synth will optimize
+      eth_tx2.tdata  = 'X; // use X so synth will optimize
-      eth_tx.tlast  = 0;
+      eth_tx2.tlast  = 0;
-      eth_tx.tuser  = '0;
+      eth_tx2.tuser  = '0;
+      eth_tx2.tkeep  = '1;
     end
-    eth_tx1_tready = eth_tx.tready;
+    eth_tx1.tready = eth_tx2.tready;
   end
 
+  //---------------------------------------
+  // Output pipeline stage
+  //---------------------------------------
+  axi4s_fifo #(
+    .SIZE(1)
+  ) in_reg_i (
+    .clear(1'b0),.space(),.occupied(),
+    .i(eth_tx2), .o(eth_tx)
+  );
+
+
 endmodule // eth_ipv4_chdr_adapter

lib/rfnoc/xport_sv/eth_ipv4_chdr_dispatch.sv

@@ -17,7 +17,8 @@
 //  Traffic not addressed (Eth) to us is dropped(optionally).
 //
 // Parameters:
-//  - CPU_FIFO_SIZE: log2 size of CPU RX fifo
+//  - CPU_FIFO_SIZE: Log2 of the FIFO depth (in bytes) for the CPU egress path
+//  - CHDR_FIFO_SIZE: Log2 of the FIFO depth (in bytes) for the CHDR egress path
 //  - PREAMBLE_BYTES: Number of bytes in the Preamble
 //  - DROP_UNKNOWN_MAC: Drop packets not addressed to us?
 //  - DROP_MIN_PACKET: Drop packets smaller than 64 bytes?
@@ -35,7 +36,8 @@
 //
 
 module eth_ipv4_chdr_dispatch #(
-  int CPU_FIFO_SIZE    = $clog2(1558),
+  int CPU_FIFO_SIZE    = $clog2(8*1024),
+  int CHDR_FIFO_SIZE   = $clog2(8*1024),
   int PREAMBLE_BYTES   = 6,
   int MAX_PACKET_BYTES = 2**16-1,
   bit DROP_UNKNOWN_MAC = 0,
@@ -51,9 +53,27 @@ module eth_ipv4_chdr_dispatch #(
   // Device addresses
   input  logic [47:0] my_mac,
   input  logic [31:0] my_ip,
-  input  logic [15:0] my_udp_chdr_port
+  input  logic [15:0] my_udp_chdr_port,
+
+  output logic        chdr_dropped,
+  output logic        cpu_dropped
 );
 
+  // Clock Crossing to the ethernet clock domain
+  logic [47:0] e_my_mac;
+  logic [31:0] e_my_ip;
+  logic [15:0] e_my_udp_chdr_port;
+  // crossing clock boundaries.
+  // my_mac, my_ip,,my_udp_chdr_port must be written
+  // prior to traffic, or an inconsistent version will
+  // exist for a clock period or 2.  This would be better
+  // done with a full handshake.
+  synchronizer #(.WIDTH(96),.STAGES(1))
+    e_info_sync (.clk(eth_rx.clk),.rst(eth_rx.rst),
+                 .in({my_mac,my_ip,my_udp_chdr_port}),
+                 .out({e_my_mac,e_my_ip,e_my_udp_chdr_port}));
+
+
   localparam ENET_USER_W = $clog2(ENET_W/8)+1;
   //---------------------------------------
   // Include for byte positions
@@ -68,6 +88,9 @@ module eth_ipv4_chdr_dispatch #(
     ``I.tready = ``O.tready;
 
   // axi_remov_bytes (PREAMBLE Strip)
+  AxiStreamIf #(.DATA_WIDTH(ENET_W),.USER_WIDTH(ENET_USER_W),
+                .TKEEP(0),.MAX_PACKET_BYTES(MAX_PACKET_BYTES))
+    inp(eth_rx.clk,eth_rx.rst);
   //   tUser = {error,trailing bytes};
   AxiStreamPacketIf #(.DATA_WIDTH(ENET_W),.USER_WIDTH(ENET_USER_W),
                 .TKEEP(0),.MAX_PACKET_BYTES(MAX_PACKET_BYTES))
@@ -86,21 +109,21 @@ module eth_ipv4_chdr_dispatch #(
   //   tUser = {error,trailing bytes};
   AxiStreamIf #(.DATA_WIDTH(ENET_W),.USER_WIDTH(ENET_USER_W),.TKEEP(0))
     cpu0(eth_rx.clk,eth_rx.rst);
-  // out_reg_cpu
+  // cpu_out_gate - throw away error packets
   //   tUser = {error,trailing bytes};
   AxiStreamIf #(.DATA_WIDTH(ENET_W),.USER_WIDTH(ENET_USER_W),.TKEEP(0))
     cpu1(eth_rx.clk,eth_rx.rst);
-  // cpu_out_gate - throw away error packets
-  //   tUser = {1'b0,trailing bytes};
-  AxiStreamIf #(.DATA_WIDTH(ENET_W),.USER_WIDTH(ENET_USER_W),.TKEEP(0))
-    cpu2(eth_rx.clk,eth_rx.rst);
   // cpu_out_fifo
   // e2c (OUTPUT)
 
   // CHDR_Branch
-  //   tUser = {error,trailing bytes};
+  //   tUser = {not used};
   AxiStreamIf #(.DATA_WIDTH(ENET_W),.TKEEP(0),.TUSER(0))
     chdr0(eth_rx.clk,eth_rx.rst);
+  //   tUser = {not used};
+  AxiStreamIf #(.DATA_WIDTH(ENET_W),.TKEEP(0),.TUSER(0))
+    chdr1(eth_rx.clk,eth_rx.rst);
+  // chdr_out_fifo
   // e2v(OUTPUT)
 
   //---------------------------------------
@@ -108,9 +131,12 @@ module eth_ipv4_chdr_dispatch #(
   //---------------------------------------
   if (PREAMBLE_BYTES > 0) begin : gen_strip_preamble
     // Strip the preamble
+    always_comb begin
+      `AXI4S_ASSIGN(inp,eth_rx);
+    end
     axi4s_remove_bytes #(.REM_START(0),.REM_END(PREAMBLE_BYTES-1)
     ) strip_preamble (
-      .i(eth_rx),.o(in0)
+      .i(inp),.o(in0)
     );
   end else begin : gen_no_preamble
     always_comb begin
@@ -143,10 +169,11 @@ module eth_ipv4_chdr_dispatch #(
   dispatch_state_t dispatch_state,next_dispatch_state  = ST_IDLE_ETH_L0;
   logic cpu_error = 1'b0;
   logic chdr_error = 1'b0;
-  logic chdr0_error = 1'b0;
+  logic mac_error;
-  logic mac_error, mac_error_old = 1'b0;
+  logic min_packet_error;
-  logic min_packet_error, min_packet_error_old = 1'b0;
   logic reached_min_packet;
+  logic cpu_push_error;
+  logic chdr_push_error;
 
   // Cached fields
   logic [47:0] eth_dst_addr_new,  eth_src_addr_new;
@@ -180,8 +207,6 @@ module eth_ipv4_chdr_dispatch #(
       udp_src_port_old  <= '0;
       udp_dst_port_old  <= '0;
       eth_type_old      <= '0;
-      mac_error_old        <= 1'b0;
-      min_packet_error_old <= 1'b0;
       reached_min_packet_old <= 1'b0;
 
       // Statemachine Decisions
@@ -204,26 +229,12 @@ module eth_ipv4_chdr_dispatch #(
 
       if (in0.tvalid && in0.tready) begin
         eth_dst_is_broadcast <= eth_dst_addr_new == ETH_ADDR_BCAST;
-        eth_dst_is_me        <= eth_dst_addr_new == my_mac;
+        eth_dst_is_me        <= eth_dst_addr_new == e_my_mac;
-        udp_dst_is_me        <= udp_dst_port_new == my_udp_chdr_port;
+        udp_dst_is_me        <= udp_dst_port_new == e_my_udp_chdr_port;
-        ipv4_dst_is_me       <= ipv4_dst_addr_new == my_ip;
+        ipv4_dst_is_me       <= ipv4_dst_addr_new == e_my_ip;
         ipv4_protocol_is_udp <= ip_protocol_new == IPV4_PROTO_UDP;
         eth_type_is_ipv4     <= eth_type_new == ETH_TYPE_IPV4;
       end
-      if (in1.tvalid && in1.tready) begin
-        if (in1.tlast) begin
-          mac_error_old <= 1'b0;
-          min_packet_error_old <= 1'b0;
-          reached_min_packet_old <= 1'b0;
-        end else begin
-          if (mac_error)
-            mac_error_old <= 1'b1;
-          if(min_packet_error)
-            min_packet_error_old <= 1'b1;
-          if(reached_min_packet_new)
-            reached_min_packet_old <= 1'b1;
-        end
-      end
     end
   end
 
@@ -240,15 +251,19 @@ module eth_ipv4_chdr_dispatch #(
     eth_type_new      = in0.get_packet_field16(eth_type_old,ETH_TYPE_BYTE,.NETWORK_ORDER(1));
   end
 
-
   always_comb begin  : reached_bytes
     reached_min_packet_new = in1.reached_packet_byte(MIN_PACKET_SIZE_BYTE);
     reached_end_of_udp     = in1.reached_packet_byte(DST_PORT_BYTE+3);// we have enough to decide
   end
-  assign mac_error          = in1.tuser[ERROR_BIT] || mac_error_old;
+
+  // calculate error conditions
+  assign mac_error          = in1.tuser[ERROR_BIT] && in1.tvalid;
+  assign cpu_push_error     = (in2.tvalid && !cpu0.tready);
+  assign chdr_push_error    = (in2.tvalid && !chdr0.tready);
+
   if (DROP_MIN_PACKET) begin
     assign reached_min_packet = (reached_min_packet_new && in1.tuser[BYTES_MSB:0] ==0) || reached_min_packet_old;
-    assign min_packet_error   = (in1.tlast && !reached_min_packet) || min_packet_error_old;
+    assign min_packet_error   = (in1.tlast && !reached_min_packet);
   end else begin
     assign reached_min_packet = 1'b1;
     assign min_packet_error   = 1'b0;
@@ -271,16 +286,16 @@ module eth_ipv4_chdr_dispatch #(
       //defaults
       next_dispatch_state = dispatch_state;
       `AXI4S_ASSIGN(in2,in1);
-      in2.tuser[ERROR_BIT] = mac_error || min_packet_error;
       cpu_error   = 1'b0;
       chdr_error  = 1'b0;
+      in1.tready  = 1'b1; // never hold off
 
       // Statemachine always returns to ST_IDLE_ETH_L0 when tlast is set
       case (dispatch_state)
         ST_IDLE_ETH_L0: begin
           cpu_error   = 1'b0;
           chdr_error  = 1'b0;
-          if (mac_error || min_packet_error) begin
+          if (mac_error || min_packet_error || cpu_push_error || chdr_push_error) begin
              cpu_error  = 1'b1;
              chdr_error = 1'b1;
              next_dispatch_state = ST_DROP_TERM;
@@ -318,7 +333,7 @@ module eth_ipv4_chdr_dispatch #(
         ST_FWD_CHDR: begin
           cpu_error   = 1'b1;
           chdr_error  = 1'b0;
-          if (mac_error || min_packet_error) begin
+          if (mac_error || min_packet_error || chdr_push_error) begin
              cpu_error  = 1'b1;
              chdr_error = 1'b1;
              next_dispatch_state = ST_DROP_TERM;
@@ -329,7 +344,7 @@ module eth_ipv4_chdr_dispatch #(
         ST_FWD_CPU: begin
           cpu_error   = 1'b0;
           chdr_error  = 1'b1;
-          if (mac_error || min_packet_error) begin
+          if (mac_error || min_packet_error || cpu_push_error) begin
              cpu_error  = 1'b1;
              chdr_error = 1'b1;
              next_dispatch_state = ST_DROP_TERM;
@@ -342,7 +357,6 @@ module eth_ipv4_chdr_dispatch #(
           chdr_error  = 1'b1;
           in2.tlast   = 1'b1;
           in2.tvalid  = 1'b1;
-          in1.tready  = in2.tready;
           next_dispatch_state = ST_DROP_WAIT;
         end
 
@@ -352,14 +366,12 @@ module eth_ipv4_chdr_dispatch #(
           chdr_error  = 1'b0;
           in2.tlast   = 1'b0;
           in2.tvalid  = 1'b0;
-          in1.tready  = 1'b1;
         end
 
         // We should never get here
         default: begin
           cpu_error   = 1'b0;
           chdr_error  = 1'b0;
-          in1.tready  = 1'b1;
           in2.tvalid  = 1'b0;
           in2.tlast   = 1'b0;
           next_dispatch_state = ST_IDLE_ETH_L0;
@@ -371,65 +383,143 @@ module eth_ipv4_chdr_dispatch #(
   //---------------------------------------
   // SPLIT
   //---------------------------------------
+  // differentiating push_errors for reporting
+  logic cpu0_push_error, cpu0_push_error_old= 1'b0;
+  logic chdr0_push_error, chdr0_push_error_old= 1'b0;
+  logic chdr0_error, chdr0_error_old = 1'b0;
+  logic cpu0_error, cpu0_error_old = 1'b0;
+
   always_comb begin : cpu0_assign
+    cpu0_error  = (cpu_error && in2.tvalid) || cpu0_error_old;
+    cpu0_push_error = (cpu_push_error && in2.tvalid)|| cpu0_push_error_old;
     cpu0.tdata = in2.tdata;
     cpu0.tuser = in2.tuser;
-    cpu0.tlast = in2.tlast;
+    cpu0.tlast = in2.tlast || cpu0_error;
-    cpu0.tvalid = in2.tvalid && chdr0.tready;
+    cpu0.tvalid = in2.tvalid || cpu0_error;
-    cpu0.tuser[ERROR_BIT]  = in2.tuser[ERROR_BIT] || cpu_error;
 
+    chdr0_error  = (chdr_error && in2.tvalid) || chdr0_error_old;
+    chdr0_push_error = (chdr_push_error && in2.tvalid) || chdr0_push_error_old;
     chdr0.tdata = in2.tdata;
     chdr0.tuser = in2.tuser;
-    chdr0.tlast = in2.tlast;
+    chdr0.tlast = in2.tlast || chdr0_error;
-    chdr0.tvalid = in2.tvalid && cpu0.tready;
+    chdr0.tvalid = in2.tvalid || chdr0_error;
-    chdr0_error  = in2.tuser[ERROR_BIT] || chdr_error;
+
+    // If the downstream sections are not ready, then the packet is dropped
+    // there isn't really any buffer up stream, so a hold off here would
+    // mean pushing back on a mac that doesn't have the capability to slow
+    // down, and a data word would be lost.
+    in2.tready = 1'b1;
 
-    in2.tready = cpu0.tready && chdr0.tready;
   end
 
-  //---------------------------------------
+  // hold the error bits until the end of the packet
-  // CPU Output processing
+  always_ff @(posedge eth_rx.clk) begin : error_ff
-  //---------------------------------------
+    if (eth_rx.rst) begin
-  axi4s_fifo #(
+      cpu0_error_old  <= 1'b0;
-    .SIZE(1)
+      chdr0_error_old <= 1'b0;
-  ) out_reg_cpu_i (
+      cpu0_push_error_old  <= 1'b0;
-    .clear(),.space(),.occupied(),
+      chdr0_push_error_old <= 1'b0;
-    .i(cpu0),.o(cpu1)
+      chdr_dropped <= 1'b0;
-  );
+      cpu_dropped  <= 1'b0;
+    end else begin
+
+      // report dropped back at the end of packet when push_error is detected.
+      // 1st term counts the drop if ready lets up before the end of the packet
+      // 2nd term counts the drop if ready is held through then end of the packet
+      // NOTE: Drop counts don't have to be perfect.  This gets pretty close though.
+      //       I.e. Don't sweat this more in the future.
+      chdr_dropped <= (chdr0_push_error && chdr0.tlast && chdr0.tvalid && chdr0.tready) ||
+                      (chdr0_push_error && in1.tlast && in1.tvalid);
+      cpu_dropped  <= (cpu0_push_error && cpu0.tlast && cpu0.tvalid && cpu0.tready) ||
+                      (cpu0_push_error && in1.tlast && in1.tvalid);
+
+      // don't clear till we discard a packet
+      if (cpu0.tlast && cpu0.tvalid && cpu0.tready) begin
+        cpu0_push_error_old  <= 1'b0;
+      // remember if we saw an error
+      end else if (cpu0_push_error && cpu0.tvalid) begin
+        cpu0_push_error_old  <= 1'b1;
+      end
+
+      // don't clear till we discard a packet
+      if (cpu0.tlast && cpu0.tvalid && cpu0.tready) begin
+        cpu0_error_old  <= 1'b0;
+      // remember if we saw an error
+      end else if (cpu0_error && cpu0.tvalid) begin
+        cpu0_error_old  <= 1'b1;
+      end
+
+      // don't clear till we discard a packet
+      if (chdr0.tlast && chdr0.tvalid && chdr0.tready) begin
+        chdr0_push_error_old  <= 1'b0;
+      // remember if we saw an error
+      end else if (chdr0_push_error && chdr0.tvalid) begin
+        chdr0_push_error_old  <= 1'b1;
+      end
+
+      // don't clear till we discard a packet
+      if (chdr0.tlast && chdr0.tvalid && chdr0.tready) begin
+        chdr0_error_old  <= 1'b0;
+      // remember if we saw an error
+      end else if (chdr0_error && chdr0.tvalid) begin
+        chdr0_error_old  <= 1'b1;
+      end
+
+    end
+  end
 
   // We cannot make a CHDR/noCHDR routing decision until we are in the middle
   // of a packet so we use a packet gate for the CPU path because we can rewind
   // the write pointer and drop the packet in case it's destined for the CHDR
-  // path.
+  // path
+
+  //---------------------------------------
+  // CPU Output processing
+  //---------------------------------------
   // NOTE: This also rejects packets with FCS failures.
   // NOTE: The SIZE of this FIFO must accommodate a 9000 byte jumbo frame
   //       regardless of the CHDR MTU
   // SIZED for 11 bit address when using a 64 bit word -> 16KByte
   // SIZED for 8 bit address when using a 512 bit word -> 16KByte
   axi4s_packet_gate #(
-    .SIZE(17-$clog2(ENET_W)), .USE_AS_BUFF(0)
+    .SIZE(14-$clog2(ENET_W/8)), .USE_AS_BUFF(0)
   ) cpu_out_gate_i (
-    .clear(1'b0), .error(cpu1.tuser[ERROR_BIT]),
+    .clear(1'b0), .error(cpu0_error),
-    .i(cpu1),.o(cpu2)
+    .i(cpu0),.o(cpu1)
   );
 
   // The CPU can be slow to respond (relative to packet wirespeed) so
   // extra buffer for packets destined there so it doesn't back up.
   axi4s_fifo #(
-    .SIZE(CPU_FIFO_SIZE)
+    .SIZE(CPU_FIFO_SIZE-$clog2(ENET_W/8))
   ) cpu_fifo_i (
     .clear(),.space(),.occupied(),
-    .i(cpu2),.o(e2c)
+    .i(cpu1),.o(e2c)
   );
 
+  //---------------------------------------
+  // CHDR Output processing
+  //---------------------------------------
   // CHDR DATA GATE
   // SIZED for 11 bit address when using a 64 bit word -> 16KByte
   // SIZED for 8 bit address when using a 512 bit word -> 16KByte
   axi4s_packet_gate #(
-    .SIZE(17-$clog2(ENET_W))
+    .SIZE(14-$clog2(ENET_W/8))
   ) chdr_out_gate_i (
     .clear(1'b0),.error(chdr0_error),
-    .i(chdr0),.o(e2v)
+    .i(chdr0),.o(chdr1)
+  );
+
+  // The transport should hook up to a crossbar downstream, which
+  // may backpressure this module because it is in the middle of
+  // transferring a packet. To ensure that upstream logic is not
+  // blocked, we instantiate at laeast one packet of buffering here.
+  // The actual size is set by CHDR_FIFO_SIZE.
+  axi4s_fifo #(
+    .SIZE(CHDR_FIFO_SIZE-$clog2(ENET_W/8))
+  ) chdr_fifo_i (
+    .clear(1'b0),.space(),.occupied(),
+    .i(chdr1),.o(e2v)
   );
 
 endmodule // eth_ipv4_chdr_dispatch

lib/rfnoc/xport_sv/eth_ipv4_interface.sv

@@ -11,14 +11,14 @@
 //
 // Parameters:
 //   - PROTOVER: RFNoC protocol version {8'd<major>, 8'd<minor>}
-//   - MTU: Log2 of the MTU of the packet in 64-bit words
+//   - CPU_FIFO_SIZE: Log2 of the FIFO depth (in bytes) for the CPU egress path
-//   - CPU_FIFO_SIZE: Log2 of the FIFO depth (in 64-bit words) for the CPU egress path
+//   - CHDR_FIFO_SIZE: Log2 of the FIFO depth (in bytes) for the CHDR egress path
 //   - RT_TBL_SIZE: Log2 of the depth of the return-address routing table
 //   - NODE_INST: The node type to return for a node-info discovery
 //   - DROP_UNKNOWN_MAC: Drop packets not addressed to us?
 //   - DROP_MIN_PACKET: Drop packets smaller than 64 bytes?
 //   - PREAMBLE_BYTES: Number of bytes of Preamble expected
-//   - ADD_SOF: Add a SOF indication into the tuser field
+//   - ADD_SOF: Add a SOF indication into the tuser field of e2c
 //   - SYNC: Set if MAC is not the same as bus_clk
 //   - ENET_W: Width of the link to the Ethernet MAC
 //   - CPU_W: Width of the CPU interface
@@ -27,8 +27,8 @@
 
 module eth_ipv4_interface #(
   logic [15:0] PROTOVER         = {8'd1, 8'd0},
-  int          MTU              = 10,
+  int          CPU_FIFO_SIZE    = $clog2(8*1024),
-  int          CPU_FIFO_SIZE    = MTU,
+  int          CHDR_FIFO_SIZE   = $clog2(8*1024),
   int          NODE_INST        = 0,
   int          RT_TBL_SIZE      = 6,
   int          REG_AWIDTH       = 14,
@@ -98,6 +98,10 @@ module eth_ipv4_interface #(
   logic [31:0] bridge_ip_reg   = DEFAULT_IP_ADDR;
   logic [15:0] bridge_udp_port = DEFAULT_UDP_PORT;
   logic        bridge_en;
+  logic        cpu_dropped;
+  logic        chdr_dropped;
+  logic [31:0] chdr_drop_count = 0;
+  logic [31:0] cpu_drop_count  = 0;
 
   always_comb begin : bridge_mux
     my_mac            = bridge_en ? bridge_mac_reg : mac_reg;
@@ -153,8 +157,16 @@ module eth_ipv4_interface #(
     if (bus_rst) begin
       reg_rd_resp <= 1'b0;
       reg_rd_data <= 32'd0;
+      chdr_drop_count <= 32'd0;
+      cpu_drop_count  <= 32'd0;
     end
     else begin
+      if (chdr_dropped) begin
+        chdr_drop_count <= chdr_drop_count+1;
+      end
+      if (cpu_dropped) begin
+        cpu_drop_count <= cpu_drop_count+1;
+      end
       if (reg_rd_req) begin
         // Assert read response one cycle after read request
         reg_rd_resp <= 1'b1;
@@ -185,7 +197,19 @@ module eth_ipv4_interface #(
 
           REG_BRIDGE_ENABLE:
             reg_rd_data <= {31'b0,bridge_en};
-
+          // Drop counts are used to debug situations
+          // Where the incoming data goes faster than
+          // chdr can consume it
+          REG_CHDR_DROPPED:
+            begin
+              reg_rd_data <= chdr_drop_count;
+              chdr_drop_count <= 0; // clear when read
+            end
+          REG_CPU_DROPPED:
+            begin
+              reg_rd_data <= cpu_drop_count;
+              cpu_drop_count <= 0; // clear when read
+            end
          default:
             reg_rd_resp <= 1'b0;
          endcase
@@ -197,10 +221,30 @@ module eth_ipv4_interface #(
     end
   end
 
+  logic b_dropped_valid;
+  logic e_cpu_dropped,  b_cpu_dropped;
+  logic e_chdr_dropped, b_chdr_dropped;
+  // push over the clock domain
+  // Sized to fit into 2 SRL's
+  axi_fifo_2clk #(.WIDTH(2), .SIZE(4)) fifo_i (
+    .reset(eth_rx.rst),
+    .i_aclk(eth_rx.clk),
+    .i_tdata({e_cpu_dropped, e_chdr_dropped}),
+    .i_tvalid(e_cpu_dropped || e_chdr_dropped), .i_tready(/*not used*/),
+    .o_aclk(bus_clk),
+    .o_tdata({b_cpu_dropped, b_chdr_dropped}),
+    .o_tvalid(b_dropped_valid), .o_tready(1'b1)
+  );
+
+  always_comb begin
+    cpu_dropped  = b_cpu_dropped  && b_dropped_valid;
+    chdr_dropped = b_chdr_dropped && b_dropped_valid;
+  end
+
   eth_ipv4_chdr_adapter #(
     .PROTOVER        (PROTOVER),
-    .MTU             (MTU),
     .CPU_FIFO_SIZE   (CPU_FIFO_SIZE),
+    .CHDR_FIFO_SIZE  (CHDR_FIFO_SIZE),
     .RT_TBL_SIZE     (RT_TBL_SIZE),
     .NODE_INST       (NODE_INST),
     .DROP_UNKNOWN_MAC(DROP_UNKNOWN_MAC),
@@ -221,7 +265,9 @@ module eth_ipv4_interface #(
     .device_id       (device_id),
     .my_mac          (my_mac   ),
     .my_ip           (my_ip    ),
-    .my_udp_chdr_port(my_udp_chdr_port  )
+    .my_udp_chdr_port(my_udp_chdr_port),
+    .chdr_dropped    (e_chdr_dropped),
+    .cpu_dropped     (e_cpu_dropped)
   );
 
 

lib/rfnoc/xport_sv/eth_regs.vh

@@ -27,3 +27,6 @@ localparam [REG_AWIDTH-1:0] REG_BRIDGE_MAC_MSB = BASE + 'h1014;
 localparam [REG_AWIDTH-1:0] REG_BRIDGE_IP      = BASE + 'h1018;
 localparam [REG_AWIDTH-1:0] REG_BRIDGE_UDP     = BASE + 'h101c;
 localparam [REG_AWIDTH-1:0] REG_BRIDGE_ENABLE  = BASE + 'h1020;
+
+localparam [REG_AWIDTH-1:0] REG_CHDR_DROPPED   = BASE + 'h1030;
+localparam [REG_AWIDTH-1:0] REG_CPU_DROPPED    = BASE + 'h1034;