From 1fcf1ce3b3e7fa637edbbcf815576d278fabb896 Mon Sep 17 00:00:00 2001 From: Wade Fife Date: Wed, 18 Aug 2021 09:38:34 -0500 Subject: [PATCH] fpga: x400: Make transport adapter width configurable Original-commit: 8e08cfe1da494f8be185470810ee19fcc8ed98d7 --- top/x400/x4xx.v | 95 ++++++++++++------- top/x400/x4xx_core.v | 147 +++++++++++++++++------------ top/x400/x4xx_qsfp_wrapper.sv | 5 +- top/x400/x4xx_qsfp_wrapper_temp.sv | 5 +- 4 files changed, 157 insertions(+), 95 deletions(-) diff --git a/top/x400/x4xx.v b/top/x400/x4xx.v index 9ff649a..4c97400 100644 --- a/top/x400/x4xx.v +++ b/top/x400/x4xx.v @@ -278,10 +278,23 @@ module x4xx ( localparam RADIO_SPC = 1; // Number of samples per cycle `endif + // Set the width for each transport adapter. All ports within a single QSFP + // will have the same width. + localparam [31:0] QSFP0_W = `CHDR_WIDTH; + localparam [31:0] QSFP1_W = `CHDR_WIDTH; + // Width of the signals used to connect the Ethernet transport adapters to + // RFNoC. Set this to the width of the widest port. + localparam [31:0] ENET_W = (QSFP0_W > QSFP1_W) ? QSFP0_W : QSFP1_W; + // Actual width of each SFP port's transport adapter interface. Each port of + // the same QSFP must have the same width. + localparam [8*32-1:0] ENET_WIDTHS = {{4{QSFP1_W}}, {4{QSFP0_W}}}; + // See global_regs_regmap_utils.vh for definition of CHDR_CLK_VALUE. localparam CHDR_CLK_RATE = CHDR_CLK_VALUE[CHDR_CLK_SIZE-1:0]; localparam RFNOC_PROTOVER = `RFNOC_PROTOVER; + localparam NET_CHDR_W = ENET_W; localparam CHDR_W = `CHDR_WIDTH; + localparam DMA_W = `CHDR_WIDTH; localparam CPU_W = 64; localparam REG_AWIDTH = 15; @@ -1539,15 +1552,15 @@ module x4xx ( wire [15:0] device_id; wire rx_rec_clk_out1; // output GTY on QSFP1 - // e2v and v2e are flattened arrays, where e2v_tdata[CHDR_W*N +: CHDR_W] is + // e2v and v2e are flattened arrays, where e2v_tdata[ENET_W*N +: ENET_W] is // the data for RFNoC port N. RFNoC ports 0-3 map to QSFP0 and ports 4-7 map - // to QSFP1. - wire [CHDR_W*8-1:0] e2v_tdata; + // to QSFP1. Note that each port might use less than ENET_W bits. + wire [ENET_W*8-1:0] e2v_tdata; wire [ 8-1:0] e2v_tlast; wire [ 8-1:0] e2v_tready; wire [ 8-1:0] e2v_tvalid; - wire [CHDR_W*8-1:0] v2e_tdata; + wire [ENET_W*8-1:0] v2e_tdata; wire [ 8-1:0] v2e_tlast; wire [ 8-1:0] v2e_tready; wire [ 8-1:0] v2e_tvalid; @@ -1640,7 +1653,8 @@ module x4xx ( .PROTOCOL3 (`QSFP0_3), `endif .CPU_W (CPU_W), - .CHDR_W (CHDR_W), + .CHDR_W (QSFP0_W), + .NET_CHDR_W (NET_CHDR_W), .BYTE_MTU (BYTE_MTU), .PORTNUM (0), .NODE_INST (0), @@ -1712,16 +1726,21 @@ module x4xx ( .rx_p (qsfp0_rx_p), .rx_n (qsfp0_rx_n), // Ethernet to CHDR - .e2v_tdata (e2v_tdata [0*CHDR_W*4 +: CHDR_W*4]), - .e2v_tlast (e2v_tlast [0* 4 +: 4]), - .e2v_tvalid (e2v_tvalid [0* 4 +: 4]), - .e2v_tready (e2v_tready [0* 4 +: 4]), + .e2v_tdata ({ e2v_tdata [3*ENET_W +: QSFP0_W], + e2v_tdata [2*ENET_W +: QSFP0_W], + e2v_tdata [1*ENET_W +: QSFP0_W], + e2v_tdata [0*ENET_W +: QSFP0_W] }), + .e2v_tlast (e2v_tlast [0 +: 4]), + .e2v_tvalid (e2v_tvalid[0 +: 4]), + .e2v_tready (e2v_tready[0 +: 4]), // CHDR to Ethernet - .v2e_tdata (v2e_tdata [0*CHDR_W*4 +: CHDR_W*4]), - .v2e_tlast (v2e_tlast [0* 4 +: 4]), - .v2e_tvalid (v2e_tvalid [0* 4 +: 4]), - .v2e_tready (v2e_tready [0* 4 +: 4]), - + .v2e_tdata ({ v2e_tdata[3*ENET_W +: QSFP0_W], + v2e_tdata[2*ENET_W +: QSFP0_W], + v2e_tdata[1*ENET_W +: QSFP0_W], + v2e_tdata[0*ENET_W +: QSFP0_W] }), + .v2e_tlast (v2e_tlast [0 +: 4]), + .v2e_tvalid (v2e_tvalid[0 +: 4]), + .v2e_tready (v2e_tready[0 +: 4]), // Misc .eth_rx_irq (eth0_rx_irq), .eth_tx_irq (eth0_tx_irq), @@ -1750,7 +1769,8 @@ module x4xx ( .PROTOCOL3 (`QSFP1_3), `endif .CPU_W (CPU_W), - .CHDR_W (CHDR_W), + .CHDR_W (QSFP1_W), + .NET_CHDR_W (NET_CHDR_W), .BYTE_MTU (BYTE_MTU), .PORTNUM (1), .NODE_INST (4), @@ -1822,15 +1842,21 @@ module x4xx ( .rx_p (qsfp1_rx_p), .rx_n (qsfp1_rx_n), // Ethernet to CHDR - .e2v_tdata (e2v_tdata [1*CHDR_W*4 +: CHDR_W*4]), - .e2v_tlast (e2v_tlast [1* 4 +: 4]), - .e2v_tvalid (e2v_tvalid [1* 4 +: 4]), - .e2v_tready (e2v_tready [1* 4 +: 4]), + .e2v_tdata ({ e2v_tdata[4*ENET_W + 3*ENET_W +: QSFP1_W], + e2v_tdata[4*ENET_W + 2*ENET_W +: QSFP1_W], + e2v_tdata[4*ENET_W + 1*ENET_W +: QSFP1_W], + e2v_tdata[4*ENET_W + 0*ENET_W +: QSFP1_W] }), + .e2v_tlast (e2v_tlast [4 +: 4]), + .e2v_tvalid (e2v_tvalid[4 +: 4]), + .e2v_tready (e2v_tready[4 +: 4]), // CHDR to Ethernet - .v2e_tdata (v2e_tdata [1*CHDR_W*4 +: CHDR_W*4]), - .v2e_tlast (v2e_tlast [1* 4 +: 4]), - .v2e_tvalid (v2e_tvalid [1* 4 +: 4]), - .v2e_tready (v2e_tready [1* 4 +: 4]), + .v2e_tdata ({ v2e_tdata[4*ENET_W + 3*ENET_W +: QSFP1_W], + v2e_tdata[4*ENET_W + 2*ENET_W +: QSFP1_W], + v2e_tdata[4*ENET_W + 1*ENET_W +: QSFP1_W], + v2e_tdata[4*ENET_W + 0*ENET_W +: QSFP1_W] }), + .v2e_tlast (v2e_tlast [4 +: 4]), + .v2e_tvalid (v2e_tvalid[4 +: 4]), + .v2e_tready (v2e_tready[4 +: 4]), // Misc .eth_rx_irq (eth1_rx_irq), .eth_tx_irq (eth1_tx_irq), @@ -1850,17 +1876,18 @@ module x4xx ( //--------------------------------------------------------------------------- // CHDR DMA bus (clk200 domain) - wire [CHDR_W-1:0] e2v_dma_tdata; - wire e2v_dma_tlast; - wire e2v_dma_tready; - wire e2v_dma_tvalid; - wire [CHDR_W-1:0] v2e_dma_tdata; - wire v2e_dma_tlast; - wire v2e_dma_tready; - wire v2e_dma_tvalid; + wire [DMA_W-1:0] e2v_dma_tdata; + wire e2v_dma_tlast; + wire e2v_dma_tready; + wire e2v_dma_tvalid; + wire [DMA_W-1:0] v2e_dma_tdata; + wire v2e_dma_tlast; + wire v2e_dma_tready; + wire v2e_dma_tvalid; eth_ipv4_internal #( - .CHDR_W (CHDR_W), + .CHDR_W (DMA_W), + .NET_CHDR_W (NET_CHDR_W), .BYTE_MTU (BYTE_MTU), .DWIDTH (REG_DWIDTH), .AWIDTH (REG_AWIDTH), @@ -2055,6 +2082,10 @@ module x4xx ( .CHDR_CLK_RATE (CHDR_CLK_RATE), .NUM_CHANNELS (NUM_CHANNELS), .CHDR_W (CHDR_W), + .DMA_W (DMA_W), + .NET_CHDR_W (NET_CHDR_W), + .ENET_W (ENET_W), + .ENET_WIDTHS (ENET_WIDTHS), .MTU (CHDR_MTU), .RFNOC_PROTOVER (RFNOC_PROTOVER), .RADIO_SPC (RADIO_SPC), diff --git a/top/x400/x4xx_core.v b/top/x400/x4xx_core.v index 13af59e..aa4b24c 100644 --- a/top/x400/x4xx_core.v +++ b/top/x400/x4xx_core.v @@ -18,6 +18,13 @@ // CHDR_CLK_RATE : rfnoc_chdr_clk rate in Hz // NUM_CHANNELS : Total number of channels // CHDR_W : CHDR width used by RFNoC +// DMA_W : Width of the DMA port interface +// NET_CHDR_W : CHDR width used by the network interface ports +// ENET_W : Width of the Ethernet buses (each port may have a unique +// width, but all signals have equal width for simplicity). +// ENET_WIDTHS : Actual width of each Ethernet port's interface. This is a +// packed array of 32-bit integers with port 0 in the +// right-most position. // MTU : Log2 of maximum transmission unit in CHDR_W sized words // RFNOC_PROTOVER : RFNoC protocol version (major[7:0], minor[7:0]) // RADIO_SPC : Number of samples per radio clock cycle @@ -26,16 +33,20 @@ module x4xx_core #( - parameter NUM_DBOARDS = 2, - parameter REG_DWIDTH = 32, - parameter REG_AWIDTH = 32, - parameter CHDR_CLK_RATE = 200000000, - parameter NUM_CHANNELS = 4, - parameter CHDR_W = 64, - parameter MTU = $clog2(8192 / (CHDR_W/8)), - parameter RFNOC_PROTOVER = {8'd1, 8'd0}, - parameter RADIO_SPC = 1, - parameter RF_BANDWIDTH = 200 + parameter NUM_DBOARDS = 2, + parameter REG_DWIDTH = 32, + parameter REG_AWIDTH = 32, + parameter CHDR_CLK_RATE = 200000000, + parameter NUM_CHANNELS = 4, + parameter CHDR_W = 64, + parameter DMA_W = 64, + parameter NET_CHDR_W = 64, + parameter [ 31:0] ENET_W = 64, + parameter [32*8-1:0] ENET_WIDTHS = {8{ENET_W}}, + parameter MTU = $clog2(8192 / (CHDR_W/8)), + parameter RFNOC_PROTOVER = {8'd1, 8'd0}, + parameter RADIO_SPC = 1, + parameter RF_BANDWIDTH = 200 ) ( // Clocks and resets input wire radio_clk, @@ -126,23 +137,23 @@ module x4xx_core #( output [ NUM_CHANNELS-1:0] tx_running, // DMA - output [CHDR_W-1:0] dmao_tdata, - output dmao_tlast, - output dmao_tvalid, - input dmao_tready, + output [DMA_W-1:0] dmao_tdata, + output dmao_tlast, + output dmao_tvalid, + input dmao_tready, - input [CHDR_W-1:0] dmai_tdata, - input dmai_tlast, - input dmai_tvalid, - output dmai_tready, + input [DMA_W-1:0] dmai_tdata, + input dmai_tlast, + input dmai_tvalid, + output dmai_tready, // e2v (Ethernet to CHDR) - output [CHDR_W*8-1:0] v2e_tdata, + output [ENET_W*8-1:0] v2e_tdata, output [ 8-1:0] v2e_tvalid, output [ 8-1:0] v2e_tlast, input [ 8-1:0] v2e_tready, // v2e (CHDR to Ethernet) - input [CHDR_W*8-1:0] e2v_tdata, + input [ENET_W*8-1:0] e2v_tdata, input [ 8-1:0] e2v_tlast, input [ 8-1:0] e2v_tvalid, output [ 8-1:0] e2v_tready, @@ -285,7 +296,7 @@ module x4xx_core #( x4xx_core_common #( .CHDR_CLK_RATE (CHDR_CLK_RATE), - .CHDR_W (CHDR_W), + .CHDR_W (NET_CHDR_W), .RFNOC_PROTOVER (RFNOC_PROTOVER), .NUM_DBOARDS (NUM_DBOARDS), .PCIE_PRESENT (0) @@ -540,8 +551,22 @@ module x4xx_core #( // Calculate how may bits wide each channel is localparam CHAN_W = 32 * RADIO_SPC; + localparam PORT_W = 512; // Set to max value; unused bits will be ignored. + localparam ENET0_W = ENET_WIDTHS[32*0 +: 32]; + localparam ENET1_W = ENET_WIDTHS[32*1 +: 32]; + localparam ENET2_W = ENET_WIDTHS[32*2 +: 32]; + localparam ENET3_W = ENET_WIDTHS[32*3 +: 32]; + localparam ENET4_W = ENET_WIDTHS[32*4 +: 32]; + rfnoc_image_core #( .CHDR_W (CHDR_W), + .PORT_W (PORT_W), + .ETH0_W (ENET0_W), + .ETH1_W (ENET1_W), + .ETH2_W (ENET2_W), + .ETH3_W (ENET3_W), + .ETH4_W (ENET4_W), + .DMA_W (DMA_W), .MTU (MTU), .PROTOVER (RFNOC_PROTOVER), .RADIO_NIPC (RADIO_SPC) @@ -631,46 +656,46 @@ module x4xx_core #( .m_axi_ruser (0), .m_axi_rvalid (dram_axi_rvalid), .m_axi_rready (dram_axi_rready), - .s_eth0_tdata (e2v_tdata [0*CHDR_W +: CHDR_W]), - .s_eth0_tlast (e2v_tlast [0* 1 +: 1]), - .s_eth0_tvalid (e2v_tvalid [0* 1 +: 1]), - .s_eth0_tready (e2v_tready [0* 1 +: 1]), - .m_eth0_tdata (v2e_tdata [0*CHDR_W +: CHDR_W]), - .m_eth0_tlast (v2e_tlast [0* 1 +: 1]), - .m_eth0_tvalid (v2e_tvalid [0* 1 +: 1]), - .m_eth0_tready (v2e_tready [0* 1 +: 1]), - .s_eth1_tdata (e2v_tdata [1*CHDR_W +: CHDR_W]), - .s_eth1_tlast (e2v_tlast [1* 1 +: 1]), - .s_eth1_tvalid (e2v_tvalid [1* 1 +: 1]), - .s_eth1_tready (e2v_tready [1* 1 +: 1]), - .m_eth1_tdata (v2e_tdata [1*CHDR_W +: CHDR_W]), - .m_eth1_tlast (v2e_tlast [1* 1 +: 1]), - .m_eth1_tvalid (v2e_tvalid [1* 1 +: 1]), - .m_eth1_tready (v2e_tready [1* 1 +: 1]), - .s_eth2_tdata (e2v_tdata [2*CHDR_W +: CHDR_W]), - .s_eth2_tlast (e2v_tlast [2* 1 +: 1]), - .s_eth2_tvalid (e2v_tvalid [2* 1 +: 1]), - .s_eth2_tready (e2v_tready [2* 1 +: 1]), - .m_eth2_tdata (v2e_tdata [2*CHDR_W +: CHDR_W]), - .m_eth2_tlast (v2e_tlast [2* 1 +: 1]), - .m_eth2_tvalid (v2e_tvalid [2* 1 +: 1]), - .m_eth2_tready (v2e_tready [2* 1 +: 1]), - .s_eth3_tdata (e2v_tdata [3*CHDR_W +: CHDR_W]), - .s_eth3_tlast (e2v_tlast [3* 1 +: 1]), - .s_eth3_tvalid (e2v_tvalid [3* 1 +: 1]), - .s_eth3_tready (e2v_tready [3* 1 +: 1]), - .m_eth3_tdata (v2e_tdata [3*CHDR_W +: CHDR_W]), - .m_eth3_tlast (v2e_tlast [3* 1 +: 1]), - .m_eth3_tvalid (v2e_tvalid [3* 1 +: 1]), - .m_eth3_tready (v2e_tready [3* 1 +: 1]), - .s_eth4_tdata (e2v_tdata [4*CHDR_W +: CHDR_W]), - .s_eth4_tlast (e2v_tlast [4* 1 +: 1]), - .s_eth4_tvalid (e2v_tvalid [4* 1 +: 1]), - .s_eth4_tready (e2v_tready [4* 1 +: 1]), - .m_eth4_tdata (v2e_tdata [4*CHDR_W +: CHDR_W]), - .m_eth4_tlast (v2e_tlast [4* 1 +: 1]), - .m_eth4_tvalid (v2e_tvalid [4* 1 +: 1]), - .m_eth4_tready (v2e_tready [4* 1 +: 1]), + .s_eth0_tdata (e2v_tdata [0*ENET_W +: ENET0_W]), + .s_eth0_tlast (e2v_tlast [0* 1 +: 1]), + .s_eth0_tvalid (e2v_tvalid [0* 1 +: 1]), + .s_eth0_tready (e2v_tready [0* 1 +: 1]), + .m_eth0_tdata (v2e_tdata [0*ENET_W +: ENET0_W]), + .m_eth0_tlast (v2e_tlast [0* 1 +: 1]), + .m_eth0_tvalid (v2e_tvalid [0* 1 +: 1]), + .m_eth0_tready (v2e_tready [0* 1 +: 1]), + .s_eth1_tdata (e2v_tdata [1*ENET_W +: ENET1_W]), + .s_eth1_tlast (e2v_tlast [1* 1 +: 1]), + .s_eth1_tvalid (e2v_tvalid [1* 1 +: 1]), + .s_eth1_tready (e2v_tready [1* 1 +: 1]), + .m_eth1_tdata (v2e_tdata [1*ENET_W +: ENET1_W]), + .m_eth1_tlast (v2e_tlast [1* 1 +: 1]), + .m_eth1_tvalid (v2e_tvalid [1* 1 +: 1]), + .m_eth1_tready (v2e_tready [1* 1 +: 1]), + .s_eth2_tdata (e2v_tdata [2*ENET_W +: ENET2_W]), + .s_eth2_tlast (e2v_tlast [2* 1 +: 1]), + .s_eth2_tvalid (e2v_tvalid [2* 1 +: 1]), + .s_eth2_tready (e2v_tready [2* 1 +: 1]), + .m_eth2_tdata (v2e_tdata [2*ENET_W +: ENET2_W]), + .m_eth2_tlast (v2e_tlast [2* 1 +: 1]), + .m_eth2_tvalid (v2e_tvalid [2* 1 +: 1]), + .m_eth2_tready (v2e_tready [2* 1 +: 1]), + .s_eth3_tdata (e2v_tdata [3*ENET_W +: ENET3_W]), + .s_eth3_tlast (e2v_tlast [3* 1 +: 1]), + .s_eth3_tvalid (e2v_tvalid [3* 1 +: 1]), + .s_eth3_tready (e2v_tready [3* 1 +: 1]), + .m_eth3_tdata (v2e_tdata [3*ENET_W +: ENET3_W]), + .m_eth3_tlast (v2e_tlast [3* 1 +: 1]), + .m_eth3_tvalid (v2e_tvalid [3* 1 +: 1]), + .m_eth3_tready (v2e_tready [3* 1 +: 1]), + .s_eth4_tdata (e2v_tdata [4*ENET_W +: ENET4_W]), + .s_eth4_tlast (e2v_tlast [4* 1 +: 1]), + .s_eth4_tvalid (e2v_tvalid [4* 1 +: 1]), + .s_eth4_tready (e2v_tready [4* 1 +: 1]), + .m_eth4_tdata (v2e_tdata [4*ENET_W +: ENET4_W]), + .m_eth4_tlast (v2e_tlast [4* 1 +: 1]), + .m_eth4_tvalid (v2e_tvalid [4* 1 +: 1]), + .m_eth4_tready (v2e_tready [4* 1 +: 1]), .s_dma_tdata (dmai_tdata), .s_dma_tlast (dmai_tlast), .s_dma_tvalid (dmai_tvalid), diff --git a/top/x400/x4xx_qsfp_wrapper.sv b/top/x400/x4xx_qsfp_wrapper.sv index 8e63028..21eac33 100644 --- a/top/x400/x4xx_qsfp_wrapper.sv +++ b/top/x400/x4xx_qsfp_wrapper.sv @@ -16,6 +16,7 @@ // lanes. See x4xx_mgt_types.vh for possible values. // CPU_W : Width of CPU interface // CHDR_W : CHDR bus width +// NET_CHDR_W : CHDR width used over the network connection // BYTE_MTU : Transport MTU in bytes // PORTNUM : Port number to distinguish multiple QSFP ports // NODE_INST : RFNoC transport adapter node instance for the first port @@ -33,6 +34,7 @@ module x4xx_qsfp_wrapper #( `MGT_Disabled}, parameter CPU_W = 64, parameter CHDR_W = 64, + parameter NET_CHDR_W = CHDR_W, parameter BYTE_MTU = $clog2(8*1024), parameter [ 7:0] PORTNUM = 8'd0, parameter NODE_INST = 0, @@ -450,7 +452,8 @@ module x4xx_qsfp_wrapper #( .PAUSE_EN (PAUSE_EN), .ENET_W (MGT_W), .CPU_W (CPU_W), - .CHDR_W (CHDR_W) + .CHDR_W (CHDR_W), + .NET_CHDR_W (NET_CHDR_W) ) eth_ipv4_interface_i ( .bus_clk (bus_clk), .bus_rst (bus_rst), diff --git a/top/x400/x4xx_qsfp_wrapper_temp.sv b/top/x400/x4xx_qsfp_wrapper_temp.sv index e503ead..2558d05 100644 --- a/top/x400/x4xx_qsfp_wrapper_temp.sv +++ b/top/x400/x4xx_qsfp_wrapper_temp.sv @@ -14,7 +14,8 @@ // PROTOCOL : Indicates the protocol to use for each of the 4 QSFP // lanes. See x4xx_mgt_types.vh for possible values. // CPU_W : Width of CPU interface -// CHDR_W : CHDR bus width +// CHDR_W : CHDR width used by RFNoC on the FPGA +// NET_CHDR_W : CHDR width used over the network connection // BYTE_MTU : Transport MTU in bytes // PORTNUM : Port number to distinguish multiple QSFP ports // NODE_INST : RFNoC transport adapter node instance for the first port @@ -31,6 +32,7 @@ module x4xx_qsfp_wrapper_temp #( parameter PROTOCOL3 = `MGT_Disabled, parameter CPU_W = 64, parameter CHDR_W = 64, + parameter NET_CHDR_W = CHDR_W, parameter BYTE_MTU = $clog2(8*1024), parameter [ 7:0] PORTNUM = 8'd0, parameter NODE_INST = 0, @@ -302,6 +304,7 @@ module x4xx_qsfp_wrapper_temp #( .PROTOCOL ({ PROTOCOL3, PROTOCOL2, PROTOCOL1, PROTOCOL0 }), .CPU_W (CPU_W), .CHDR_W (CHDR_W), + .NET_CHDR_W (NET_CHDR_W), .BYTE_MTU (BYTE_MTU), .PORTNUM (PORTNUM), .NODE_INST (NODE_INST),