Martin Braun
ec4fab0340
- 2to3 was used to convert the Python scripts, except where the tool choked and manual intervention was required - All references to "python" where replaced with "python3" - buffer() was replaced by memoryview() Original-commit: ca68195b5d12c5410cfac8d459a0b0902c4c72c7
594 lines
32 KiB
Python
Executable File
594 lines
32 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
#
|
|
# Copyright 2016 Ettus Research
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
#
|
|
|
|
import math
|
|
import rfnocsim
|
|
import ni_hw_models as hw
|
|
|
|
class ColGlobals():
|
|
BPI = 4 # Number of bytes per sample or coefficient
|
|
BPP = 1024 # Bytes per packet
|
|
MIN_SAMP_HOPS = 1 # Minimum number of hops an RX sample will take before it is used to compute a PP
|
|
MAX_SAMP_HOPS = 3 # Maximum number of hops an RX sample will take before it is used to compute a PP
|
|
MIN_PP_HOPS = 0 # Minimum number of hops a PP will take before it is used to compute a TX sample
|
|
MAX_PP_HOPS = 1 # Maximum number of hops a PP will take before it is used to compute a TX sample
|
|
ELASTIC_BUFF_FULLNESS = 0.5
|
|
|
|
class PartialContribComputer(rfnocsim.Function):
|
|
"""
|
|
Simulation model for function that computes the contribution of radio chans on other radio chans.
|
|
This function computes a NxM dot product of FFTs, one bin at a time.
|
|
Features:
|
|
- Supports computing the product in multiple cycles (for resource reuse)
|
|
- Supports deinterleaving data in streams (i.e. is Radio 0+1 data comes in thru the same ethernet)
|
|
|
|
Args:
|
|
sim_core: Simulator core object
|
|
name: Name of this function
|
|
size: Number of chans (inputs) for which contribution partial products are computed
|
|
fft_size: The length of the FFT in bins
|
|
dst_chans: Computes the contribution of the input chans on these dst_chans
|
|
items_per_stream: How many channels per stream can this function deinterleave?
|
|
ticks_per_exec: How many ticks for the function to generate a full output set
|
|
"""
|
|
def __init__(self, sim_core, name, size, dst_chans, items_per_stream, app_settings):
|
|
ticks_per_exec = 1 # This function will run once every tick. No multi-cycle paths here.
|
|
rfnocsim.Function.__init__(self, sim_core, name, size, int(len(dst_chans)/items_per_stream), ticks_per_exec)
|
|
self.items_per_stream = items_per_stream # Each stream contains data from n radio chans
|
|
self.dst_chans = dst_chans # Where should the individual products go?
|
|
# This block has to buffer enough data to ensure
|
|
# sample alignment. How deep should those buffers be?
|
|
sync_buff_depth = (((ColGlobals.MAX_SAMP_HOPS - ColGlobals.MIN_SAMP_HOPS) *
|
|
hw.Bee7Fpga.IO_LN_LATENCY * float(app_settings['samp_rate'])) / ColGlobals.ELASTIC_BUFF_FULLNESS)
|
|
|
|
# Adder latency: log2(radix) adder stages + 2 pipeline flops
|
|
latency = math.ceil(math.log(size/len(dst_chans), 2)) + 2
|
|
# Synchronization latency based on buffer size
|
|
latency += (sync_buff_depth * ColGlobals.ELASTIC_BUFF_FULLNESS) * (self.get_tick_rate() / float(app_settings['samp_rate']))
|
|
# Packet alignment latency
|
|
latency += ColGlobals.BPP * (self.get_tick_rate() / hw.Bee7Fpga.IO_LN_BW)
|
|
self.estimate_resources(size*items_per_stream, len(dst_chans), app_settings, sync_buff_depth*size, latency)
|
|
|
|
def estimate_resources(self, N, M, app_settings, sync_buff_total_samps, pre_filt_latency):
|
|
rscrs = rfnocsim.HwRsrcs()
|
|
|
|
DSP_BLOCKS_PER_MAC = 3 # DSP blocks for a scaled complex MAC
|
|
MAX_DSP_RATE = 400e6 # Max clock rate for a DSP48E block
|
|
MAX_UNROLL_DEPTH = 2 # How many taps (or FFT bins) to compute in parallel?
|
|
COEFF_SETS = 1 # We need two copies of coefficients one live
|
|
# and one buffered for dynamic reload. If both
|
|
# live in BRAM, this should be 2. If the live
|
|
# set lives in registers, this should be 1
|
|
|
|
samp_rate = float(app_settings['samp_rate'])
|
|
dsp_cyc_per_samp = MAX_DSP_RATE / samp_rate
|
|
|
|
if app_settings['domain'] == 'time':
|
|
fir_taps = app_settings['fir_taps']
|
|
if (fir_taps <= dsp_cyc_per_samp):
|
|
unroll_factor = 1
|
|
dsp_rate = samp_rate * fir_taps
|
|
else:
|
|
unroll_factor = math.ceil((1.0 * fir_taps) / dsp_cyc_per_samp)
|
|
dsp_rate = MAX_DSP_RATE
|
|
if (unroll_factor > MAX_UNROLL_DEPTH):
|
|
raise self.SimCompError('Too many FIR coefficients! Reached loop unroll limit.')
|
|
|
|
rscrs.add('DSP', DSP_BLOCKS_PER_MAC * unroll_factor * N * M)
|
|
rscrs.add('BRAM_18kb', math.ceil(ColGlobals.BPI * app_settings['fir_dly_line'] / hw.Bee7Fpga.BRAM_BYTES) * N * M) # FIR delay line memory
|
|
rscrs.add('BRAM_18kb', math.ceil(ColGlobals.BPI * COEFF_SETS * fir_taps * unroll_factor * N * M / hw.Bee7Fpga.BRAM_BYTES)) # Coefficient storage
|
|
|
|
samp_per_tick = dsp_rate / self.get_tick_rate()
|
|
self.update_latency(func=pre_filt_latency + (fir_taps / (samp_per_tick * unroll_factor)))
|
|
else:
|
|
fft_size = app_settings['fft_size']
|
|
rscrs.add('DSP', DSP_BLOCKS_PER_MAC * N * M * MAX_UNROLL_DEPTH) # MACs
|
|
rscrs.add('BRAM_18kb', math.ceil(ColGlobals.BPI * N * M * fft_size * COEFF_SETS / hw.Bee7Fpga.BRAM_BYTES)) # Coeff storage
|
|
|
|
samp_per_tick = MAX_DSP_RATE / self.get_tick_rate()
|
|
self.update_latency(func=pre_filt_latency + (fft_size / samp_per_tick))
|
|
|
|
rscrs.add('BRAM_18kb', math.ceil(ColGlobals.BPI * sync_buff_total_samps / hw.Bee7Fpga.BRAM_BYTES))
|
|
self.update_rsrcs(rscrs)
|
|
|
|
def do_func(self, in_data):
|
|
"""
|
|
Gather FFT data from "size" channels, compute a dot product with the coeffieicnt
|
|
matrix and spit the partial products out. The dot product is computed for each
|
|
FFT bin serially.
|
|
"""
|
|
out_data = list()
|
|
src_chans = []
|
|
# Iterate over each input
|
|
for di in in_data:
|
|
if len(di.items) != self.items_per_stream:
|
|
raise RuntimeError('Incorrect items per stream. Expecting ' + str(self.items_per_stream))
|
|
# Deinterleave data
|
|
for do in range(len(di.items)):
|
|
(sid, coords) = rfnocsim.DataStream.submatrix_parse(di.items[do])
|
|
if sid != 'rx':
|
|
raise RuntimeError('Incorrect items. Expecting radio data (rx) but got ' + sid)
|
|
src_chans.extend(coords[0])
|
|
bpi = in_data[0].bpi
|
|
count = in_data[0].count
|
|
# Iterate through deinterleaved channels
|
|
for i in range(0, len(self.dst_chans), self.items_per_stream):
|
|
items = []
|
|
for j in range(self.items_per_stream):
|
|
# Compute partial products:
|
|
# pp = partial product of "src_chans" on "self.dst_chans[i+j]"
|
|
items.append(rfnocsim.DataStream.submatrix_gen('pp', [src_chans, self.dst_chans[i+j]]))
|
|
out_data.append(self.create_outdata_stream(bpi, items, count))
|
|
return out_data
|
|
|
|
class PartialContribCombiner(rfnocsim.Function):
|
|
"""
|
|
Simulation model for function that adds multiple partial contributions (products) into a larger
|
|
partial product. The combiner can optionally reduce a very large product into a smaller one.
|
|
Ex: pp[31:0,i] (contribution on chan 0..31 on i) can alias to tx[i] if there are 32 channels.
|
|
|
|
Args:
|
|
sim_core: Simulator core object
|
|
name: Name of this function
|
|
radix: Number of partial products that are combined (Number of inputs)
|
|
reducer_filter: A tuple that represents what pp channels to alias to what
|
|
items_per_stream: How many channels per stream can this function deinterleave?
|
|
"""
|
|
|
|
def __init__(self, sim_core, name, radix, app_settings, reducer_filter = (None, None), items_per_stream = 2):
|
|
rfnocsim.Function.__init__(self, sim_core, name, radix, 1)
|
|
self.radix = radix
|
|
self.reducer_filter = reducer_filter
|
|
self.items_per_stream = items_per_stream
|
|
|
|
# This block has to buffer enough data to ensure
|
|
# sample alignment. How deep should those buffers be?
|
|
sync_buff_depth = (((ColGlobals.MAX_PP_HOPS - ColGlobals.MIN_PP_HOPS) *
|
|
hw.Bee7Fpga.IO_LN_LATENCY * float(app_settings['samp_rate'])) / ColGlobals.ELASTIC_BUFF_FULLNESS)
|
|
# Figure out latency based on sync buffer and delay line
|
|
latency = math.ceil(math.log(radix, 2)) + 2 # log2(radix) adder stages + 2 pipeline flops
|
|
# Synchronization latency based on buffer size
|
|
latency += (sync_buff_depth * ColGlobals.ELASTIC_BUFF_FULLNESS) * (self.get_tick_rate() / float(app_settings['samp_rate']))
|
|
# Packet alignment latency
|
|
latency += ColGlobals.BPP * (self.get_tick_rate() / hw.Bee7Fpga.IO_LN_BW)
|
|
|
|
self.update_latency(func=latency)
|
|
self.estimate_resources(radix, sync_buff_depth)
|
|
|
|
def estimate_resources(self, radix, sync_buff_depth):
|
|
rscrs = rfnocsim.HwRsrcs()
|
|
# Assume that pipelined adders are inferred in logic (not DSP)
|
|
# Assume that buffering uses BRAM
|
|
rscrs.add('BRAM_18kb', math.ceil(ColGlobals.BPI * sync_buff_depth * radix / hw.Bee7Fpga.BRAM_BYTES))
|
|
self.update_rsrcs(rscrs)
|
|
|
|
def do_func(self, in_data):
|
|
"""
|
|
Gather partial dot products from inputs, add them together and spit them out
|
|
Perform sanity check to ensure that we are adding the correct things
|
|
"""
|
|
out_chans = dict()
|
|
# Iterate over each input
|
|
for di in in_data:
|
|
if len(di.items) != self.items_per_stream:
|
|
raise self.SimCompError('Incorrect items per stream. Expecting ' + str(self.items_per_stream))
|
|
# Deinterleave data
|
|
for do in range(len(di.items)):
|
|
(sid, coords) = rfnocsim.DataStream.submatrix_parse(di.items[do])
|
|
if sid == 'null':
|
|
continue
|
|
elif sid != 'pp':
|
|
raise self.SimCompError('Incorrect items. Expecting partial produts (pp) but got ' + sid)
|
|
if len(coords[1]) != 1:
|
|
raise self.SimCompError('Incorrect partial product. Target must be a single channel')
|
|
if coords[1][0] in out_chans:
|
|
out_chans[coords[1][0]].extend(coords[0])
|
|
else:
|
|
out_chans[coords[1][0]] = coords[0]
|
|
# Check if keys (targets) for partial products == items_per_stream
|
|
if len(list(out_chans.keys())) != self.items_per_stream:
|
|
raise self.SimCompError('Inconsistent partial products. Too many targets.')
|
|
# Verify that all influencers for each target are consistent
|
|
if not all(x == list(out_chans.values())[0] for x in list(out_chans.values())):
|
|
raise self.SimCompError('Inconsistent partial products. Influencers dont match.')
|
|
contrib_chans = list(out_chans.values())[0]
|
|
# Combine partial products and return
|
|
out_items = []
|
|
for ch in list(out_chans.keys()):
|
|
if sorted(self.reducer_filter[0]) == sorted(contrib_chans):
|
|
out_items.append(rfnocsim.DataStream.submatrix_gen(self.reducer_filter[1], [ch]))
|
|
else:
|
|
out_items.append(rfnocsim.DataStream.submatrix_gen('pp', [list(out_chans.values())[0], ch]))
|
|
return self.create_outdata_stream(in_data[0].bpi, out_items, in_data[0].count)
|
|
|
|
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
|
# NOTE: The Torus Topology has not been maintained. Use at your own risk
|
|
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
|
class Topology_2D_4x4_Torus:
|
|
@classmethod
|
|
def config_bitstream(cls, bee7fpga, app_settings, in_chans, out_chans, total_num_chans, is_radio_node):
|
|
if len(in_chans) != 64:
|
|
raise bee7fpga.SimCompError('in_chans must be 64 channels wide. Got ' + str(len(in_chans)))
|
|
if len(out_chans) != 16:
|
|
raise bee7fpga.SimCompError('out_chans must be 16 channels wide. Got ' + str(len(out_chans)))
|
|
GRP_LEN = 16 / 2 # 2 radio channesl per USRP
|
|
|
|
# Broadcast raw data streams to all internal and external FPGAs
|
|
for i in range(GRP_LEN):
|
|
in_ln = bee7fpga.EXT_IO_LANES[bee7fpga.BP_BASE+i]
|
|
bee7fpga.sim_core.connect(bee7fpga.serdes_i[in_ln], 0, bee7fpga.serdes_o[bee7fpga.EW_IO_LANES[i]], 0)
|
|
bee7fpga.sim_core.connect(bee7fpga.serdes_i[in_ln], 0, bee7fpga.serdes_o[bee7fpga.NS_IO_LANES[i]], 0)
|
|
bee7fpga.sim_core.connect(bee7fpga.serdes_i[in_ln], 0, bee7fpga.serdes_o[bee7fpga.XX_IO_LANES[i]], 0)
|
|
bee7fpga.sim_core.connect(bee7fpga.serdes_i[in_ln], 0, bee7fpga.serdes_o[bee7fpga.EXT_IO_LANES[bee7fpga.BP_BASE+8+i]], 0)
|
|
# Create an internal bus to hold the generated partial products
|
|
bee7fpga.pp_bus = dict()
|
|
for i in range(GRP_LEN):
|
|
bee7fpga.pp_bus[i] = rfnocsim.Channel(bee7fpga.sim_core, '%s/_INTERNAL_PP_%02d' % (bee7fpga.name,i))
|
|
# We need to compute partial products of the data that is broadcast to us
|
|
# pp_input_lanes represents the IO lanes that hold this data
|
|
pp_input_lanes = bee7fpga.EXT_IO_LANES[bee7fpga.BP_BASE:bee7fpga.BP_BASE+GRP_LEN] + \
|
|
bee7fpga.EW_IO_LANES[0:GRP_LEN] + bee7fpga.NS_IO_LANES[0:GRP_LEN] + bee7fpga.XX_IO_LANES[0:GRP_LEN]
|
|
# The function that computes the partial products
|
|
func = PartialContribComputer(
|
|
sim_core=bee7fpga.sim_core, name=bee7fpga.name + '/pp_computer/', size=len(pp_input_lanes),
|
|
dst_chans=out_chans,
|
|
items_per_stream=2, app_settings=app_settings)
|
|
for i in range(len(pp_input_lanes)):
|
|
bee7fpga.sim_core.connect(bee7fpga.serdes_i[pp_input_lanes[i]], 0, func, i)
|
|
for i in range(GRP_LEN): #Outputs of function
|
|
bee7fpga.sim_core.connect(func, i, bee7fpga.pp_bus[i], 0)
|
|
bee7fpga.add_function(func)
|
|
# Add a function combine all partial products (one per IO lane)
|
|
for i in range(GRP_LEN):
|
|
func = PartialContribCombiner(
|
|
sim_core=bee7fpga.sim_core, name=bee7fpga.name + '/pp_combiner_%d/' % (i),
|
|
radix=2, app_settings=app_settings, reducer_filter=(list(range(total_num_chans)), 'tx'))
|
|
# Partial products generated internally have to be added to a partial
|
|
# sum coming from outside
|
|
bee7fpga.sim_core.connect(bee7fpga.serdes_i[bee7fpga.EXT_IO_LANES[bee7fpga.FP_BASE+i]], 0, func, 0)
|
|
bee7fpga.sim_core.connect(bee7fpga.pp_bus[i], 0, func, 1)
|
|
# If this FPGA is hooked up to the radio then send partial products
|
|
# back to when samples came from. Otherwise send it out to the PP output bus
|
|
if is_radio_node:
|
|
bee7fpga.sim_core.connect(func, 0, bee7fpga.serdes_o[bee7fpga.EXT_IO_LANES[bee7fpga.BP_BASE+i]], 0)
|
|
else:
|
|
bee7fpga.sim_core.connect(func, 0, bee7fpga.serdes_o[bee7fpga.EXT_IO_LANES[bee7fpga.FP_BASE+8+i]], 0)
|
|
bee7fpga.add_function(func)
|
|
|
|
@classmethod
|
|
def connect(cls, sim_core, usrps, bee7blades, hosts, app_settings):
|
|
USRPS_PER_BLADE = 32
|
|
|
|
# Create NULL source of "zero" partial products
|
|
null_items = ['null[(0);(0)]', 'null[(0);(0)]']
|
|
null_src = rfnocsim.Producer(sim_core, 'NULL_SRC', 4, null_items)
|
|
if app_settings['domain'] == 'frequency':
|
|
null_src.set_rate(app_settings['samp_rate']*(1.0 +
|
|
(float(app_settings['fft_overlap'])/app_settings['fft_size'])))
|
|
else:
|
|
null_src.set_rate(app_settings['samp_rate'])
|
|
|
|
# Reshape BEE7s
|
|
# The blades are arranged in 2D Torus network with 4 blades across
|
|
# each dimension (4x4 = 16)
|
|
bee7grid = []
|
|
for r in range(4):
|
|
bee7row = []
|
|
for c in range(4):
|
|
blade = bee7blades[4*r + c]
|
|
pp_chans = list(range(64*c,64*(c+1)))
|
|
for i in range(4):
|
|
Topology_2D_4x4_Torus.config_bitstream(
|
|
blade.fpgas[i], app_settings, pp_chans, pp_chans[i*16:(i+1)*16], 256, (r==c))
|
|
bee7row.append(blade)
|
|
bee7grid.append(bee7row)
|
|
|
|
# USRP-Bee7 Connections
|
|
# Blades across the diagonal are connected to USRPs
|
|
for b in range(4):
|
|
for u in range(USRPS_PER_BLADE):
|
|
sim_core.connect_bidir(
|
|
usrps[USRPS_PER_BLADE*b + u], 0, bee7grid[b][b],
|
|
len(hw.Bee7Fpga.EXT_IO_LANES)*(u/8) + hw.Bee7Fpga.BP_BASE+(u%8), 'SAMP')
|
|
sim_core.connect_bidir(
|
|
hosts[b], 0, bee7grid[b][b], hw.Bee7Fpga.FP_BASE+8, 'CONFIG', ['blue','blue'])
|
|
|
|
# Bee7-Bee7 Connections
|
|
null_srcs = []
|
|
for r in range(4): # Traverse across row
|
|
for c in range(4): # Traverse across col
|
|
for f in range(4):
|
|
samp_in_base = len(hw.Bee7Fpga.EXT_IO_LANES)*f + hw.Bee7Fpga.BP_BASE
|
|
samp_out_base = len(hw.Bee7Fpga.EXT_IO_LANES)*f + hw.Bee7Fpga.BP_BASE+8
|
|
pp_in_base = len(hw.Bee7Fpga.EXT_IO_LANES)*f + hw.Bee7Fpga.FP_BASE
|
|
pp_out_base = len(hw.Bee7Fpga.EXT_IO_LANES)*f + hw.Bee7Fpga.FP_BASE+8
|
|
if r != c:
|
|
sim_core.connect_multi_bidir(
|
|
bee7grid[r][(c+3)%4], list(range(samp_out_base,samp_out_base+8)),
|
|
bee7grid[r][c], list(range(samp_in_base,samp_in_base+8)),
|
|
'SAMP_O2I', ['black','blue'])
|
|
sim_core.connect_multi_bidir(
|
|
bee7grid[r][c], list(range(pp_out_base,pp_out_base+8)),
|
|
bee7grid[(r+1)%4][c], list(range(pp_in_base,pp_in_base+8)),
|
|
'PP_O2I', ['black','blue'])
|
|
else:
|
|
for i in range(8):
|
|
sim_core.connect(null_src, 0, bee7grid[(r+1)%4][c], pp_in_base + i)
|
|
|
|
class Topology_3D_4x4_FLB:
|
|
@classmethod
|
|
def get_radio_num(cls, router_addr, radio_idx, concentration):
|
|
"""
|
|
Returns the global radio index given local radio info
|
|
|
|
(global_radio_idx) = get_radio_num(router_addr, radio_idx, concentration) where:
|
|
- router_addr: Address of the current FPGA (router) in 3-D space
|
|
- radio_idx: The local index of the radio for the current router_addr
|
|
- concentration: Number of USRPs connected to each router
|
|
"""
|
|
DIM_SIZE = 4
|
|
multiplier = concentration
|
|
radio_num = 0
|
|
for dim in ['Z','Y','X']:
|
|
radio_num += router_addr[dim] * multiplier
|
|
multiplier *= DIM_SIZE
|
|
return radio_num + radio_idx
|
|
|
|
@classmethod
|
|
def get_portmap(cls, node_addr):
|
|
"""
|
|
Returns the router and terminal connections for the current FPGA
|
|
|
|
(router_map, terminal_map) = get_portmap(node_addr) where:
|
|
- node_addr: Address of the current FPGA in 3-D space
|
|
- router_map: A double map indexed by the dimension {X,Y,Z} and the
|
|
FPGA address in that dimension that returns the Aurora
|
|
lane index that connects the current node to the neighbor.
|
|
Example: if node_addr = [0,0,0] then router_map['X'][1] will
|
|
hold the IO lane index that connects the current node with
|
|
its X-axis neighbor with address 1
|
|
- terminal_map: A single map that maps a dimension {X,Y,Z} to the starting
|
|
IO lane index for terminals (like USRPs) in that dimension.
|
|
A terminal is a leaf node in the network.
|
|
"""
|
|
router_map = dict()
|
|
terminal_map = dict()
|
|
# If "node_addr" is the address of the current FPGA in the (X,Y,Z) space,
|
|
# then build a list of other addresses (neighbors) in each dimension
|
|
DIM_SIZE = 4
|
|
for dim in ['X','Y','Z']:
|
|
all_addrs = list(range(DIM_SIZE))
|
|
all_addrs.remove(node_addr[dim])
|
|
router_map[dim] = dict()
|
|
for dst in all_addrs:
|
|
router_map[dim][dst] = 0 # Assign lane index as 0 for now
|
|
# Assign Aurora lanes for all external connections between BEE7s
|
|
io_base = hw.Bee7Fpga.EXT_IO_LANES[0]
|
|
|
|
# ---- X-axis ----
|
|
# All BEE7s in the X dimension are connected via the RTM
|
|
# The fist quad on the RTM is reserved for SFP+ peripherals like
|
|
# the USRPs, Ethernet switch ports, etc
|
|
# All others are used for inter BEE connections over QSFP+
|
|
terminal_map['X'] = io_base + hw.Bee7Fpga.BP_BASE
|
|
xdst = terminal_map['X'] + DIM_SIZE
|
|
for dst in router_map['X']:
|
|
router_map['X'][dst] = xdst
|
|
xdst += DIM_SIZE
|
|
|
|
# ---- Z-axis ----
|
|
# All BEE7s in the Z dimension are connected via FMC IO cards (front panel)
|
|
# To be symmetric with the X-axis the first quad on the FMC bus is also
|
|
# reserved (regardless of all quads being symmetric)
|
|
terminal_map['Z'] = io_base + hw.Bee7Fpga.FP_BASE
|
|
zdst = terminal_map['Z'] + DIM_SIZE
|
|
for dst in router_map['Z']:
|
|
router_map['Z'][dst] = zdst
|
|
zdst += DIM_SIZE
|
|
|
|
# ---- Y-axis ----
|
|
# Within a BEE7, FPGAs re connected in the Y-dimension:
|
|
# 0 - 1
|
|
# | X |
|
|
# 2 - 3
|
|
Y_LANE_MAP = {
|
|
0:{1:hw.Bee7Fpga.EW_IO_LANES[0], 2:hw.Bee7Fpga.NS_IO_LANES[0], 3:hw.Bee7Fpga.XX_IO_LANES[0]},
|
|
1:{0:hw.Bee7Fpga.EW_IO_LANES[0], 2:hw.Bee7Fpga.XX_IO_LANES[0], 3:hw.Bee7Fpga.NS_IO_LANES[0]},
|
|
2:{0:hw.Bee7Fpga.NS_IO_LANES[0], 1:hw.Bee7Fpga.XX_IO_LANES[0], 3:hw.Bee7Fpga.EW_IO_LANES[0]},
|
|
3:{0:hw.Bee7Fpga.XX_IO_LANES[0], 1:hw.Bee7Fpga.NS_IO_LANES[0], 2:hw.Bee7Fpga.EW_IO_LANES[0]}}
|
|
for dst in router_map['Y']:
|
|
router_map['Y'][dst] = Y_LANE_MAP[node_addr['Y']][dst]
|
|
|
|
return (router_map, terminal_map)
|
|
|
|
@classmethod
|
|
def config_bitstream(cls, bee7fpga, app_settings, fpga_addr):
|
|
"""
|
|
Defines the FPGA behavior for the current FPGA. This function will make
|
|
create the necessary simulation functions, connect them to IO lanes and
|
|
define the various utilization metrics for the image.
|
|
|
|
config_bitstream(bee7fpga, app_settings, fpga_addr):
|
|
- bee7fpga: The FPGA simulation object being configured
|
|
- fpga_addr: Address of the FPGA in 3-D space
|
|
- app_settings: Application information
|
|
"""
|
|
if len(fpga_addr) != 3:
|
|
raise bee7fpga.SimCompError('fpga_addr must be 3-dimensional. Got ' + str(len(fpga_addr)))
|
|
|
|
# Map that stores lane indices for all neighbors of this node
|
|
(router_map, terminal_map) = cls.get_portmap(fpga_addr)
|
|
# USRPs are connected in the X dimension (RTM) because it has SFP+ ports
|
|
base_usrp_lane = terminal_map['X']
|
|
|
|
DIM_WIDTH = 4 # Dimension size for the 3-D network
|
|
MAX_USRPS = 4 # Max USRPs that can possibly be connected to each FPGA
|
|
NUM_USRPS = 2 # Number of USRPs actually connected to each FPGA
|
|
CHANS_PER_USRP = 2 # How many radio channels does each USRP have
|
|
ALL_CHANS = list(range(pow(DIM_WIDTH, 3) * NUM_USRPS * CHANS_PER_USRP))
|
|
|
|
# Each FPGA will forward the sample stream from each USRP to all of its
|
|
# X-axis neighbors
|
|
for ri in router_map['X']:
|
|
for li in range(MAX_USRPS): # li = GT Lane index
|
|
bee7fpga.sim_core.connect(bee7fpga.serdes_i[base_usrp_lane + li], 0, bee7fpga.serdes_o[router_map['X'][ri] + li], 0)
|
|
|
|
# Consequently, this FPGA will receive the USRP sample streams from each of
|
|
# its X-axis neighbors. Define an internal bus to aggregate all the neighbor
|
|
# streams with the native ones. Order the streams such that each FPGA sees the
|
|
# same data streams.
|
|
bee7fpga.int_samp_bus = dict()
|
|
for i in range(DIM_WIDTH):
|
|
for li in range(MAX_USRPS): # li = GT Lane index
|
|
bee7fpga.int_samp_bus[(MAX_USRPS*i) + li] = rfnocsim.Channel(
|
|
bee7fpga.sim_core, '%s/_INT_SAMP_%02d' % (bee7fpga.name,(MAX_USRPS*i) + li))
|
|
ln_base = base_usrp_lane if i == fpga_addr['X'] else router_map['X'][i]
|
|
bee7fpga.sim_core.connect(bee7fpga.serdes_i[ln_base + li], 0, bee7fpga.int_samp_bus[(MAX_USRPS*i) + li], 0)
|
|
|
|
# Forward the X-axis aggregated sample streams to all Y-axis neighbors
|
|
for ri in router_map['Y']:
|
|
for li in range(DIM_WIDTH*DIM_WIDTH): # li = GT Lane index
|
|
bee7fpga.sim_core.connect(bee7fpga.int_samp_bus[li], 0, bee7fpga.serdes_o[router_map['Y'][ri] + li], 0)
|
|
|
|
# What partial products will this FPGA compute?
|
|
# Generate channel list to compute partial products
|
|
pp_chans = list()
|
|
for cg in range(DIM_WIDTH): # cg = Channel group
|
|
for r in range(NUM_USRPS):
|
|
radio_num = cls.get_radio_num({'X':fpga_addr['X'], 'Y':fpga_addr['Y'], 'Z':cg}, r, NUM_USRPS)
|
|
for ch in range(CHANS_PER_USRP):
|
|
pp_chans.append(radio_num*CHANS_PER_USRP + ch)
|
|
|
|
# Instantiate partial product computer
|
|
bee7fpga.func_pp_comp = PartialContribComputer(
|
|
sim_core=bee7fpga.sim_core, name=bee7fpga.name+'/pp_computer/', size=DIM_WIDTH*DIM_WIDTH*NUM_USRPS,
|
|
dst_chans=pp_chans,
|
|
items_per_stream=CHANS_PER_USRP, app_settings=app_settings)
|
|
bee7fpga.add_function(bee7fpga.func_pp_comp)
|
|
|
|
# Partial product computer takes inputs from all Y-axis links
|
|
for sg in range(DIM_WIDTH): # sg = Group of sexdectects
|
|
for qi in range(DIM_WIDTH): # qi = GT Quad index
|
|
for li in range(NUM_USRPS):
|
|
func_inln = (sg * DIM_WIDTH * NUM_USRPS) + (qi * NUM_USRPS) + li
|
|
if sg == fpga_addr['Y']:
|
|
bee7fpga.sim_core.connect(bee7fpga.int_samp_bus[(qi * DIM_WIDTH) + li], 0,
|
|
bee7fpga.func_pp_comp, func_inln)
|
|
else:
|
|
bee7fpga.sim_core.connect(bee7fpga.serdes_i[router_map['Y'][sg] + (qi * DIM_WIDTH) + li], 0,
|
|
bee7fpga.func_pp_comp, func_inln)
|
|
|
|
# Internal bus to hold aggregated partial products
|
|
bee7fpga.pp_bus = dict()
|
|
for i in range(DIM_WIDTH*NUM_USRPS):
|
|
bee7fpga.pp_bus[i] = rfnocsim.Channel(bee7fpga.sim_core, '%s/_INT_PP_%02d' % (bee7fpga.name,i))
|
|
bee7fpga.sim_core.connect(bee7fpga.func_pp_comp, i, bee7fpga.pp_bus[i], 0)
|
|
|
|
# Forward partial products to Z-axis neighbors
|
|
for ri in router_map['Z']:
|
|
for li in range(NUM_USRPS): # li = GT Lane index
|
|
bee7fpga.sim_core.connect(bee7fpga.pp_bus[ri*NUM_USRPS + li], 0, bee7fpga.serdes_o[router_map['Z'][ri] + li], 0)
|
|
|
|
# Instantiate partial product adder
|
|
bee7fpga.func_pp_comb = dict()
|
|
for i in range(NUM_USRPS):
|
|
bee7fpga.func_pp_comb[i] = PartialContribCombiner(
|
|
sim_core=bee7fpga.sim_core, name=bee7fpga.name + '/pp_combiner_%d/'%(i),
|
|
radix=DIM_WIDTH, app_settings=app_settings, reducer_filter=(ALL_CHANS, 'tx'),
|
|
items_per_stream=CHANS_PER_USRP)
|
|
bee7fpga.add_function(bee7fpga.func_pp_comb[i])
|
|
|
|
# Aggregate partial products from Z-axis neighbors
|
|
for u in range(NUM_USRPS):
|
|
for ri in range(DIM_WIDTH):
|
|
if ri in router_map['Z']:
|
|
bee7fpga.sim_core.connect(bee7fpga.serdes_i[router_map['Z'][ri] + u], 0, bee7fpga.func_pp_comb[u], ri)
|
|
else:
|
|
bee7fpga.sim_core.connect(bee7fpga.pp_bus[ri*NUM_USRPS + u], 0, bee7fpga.func_pp_comb[u], ri)
|
|
|
|
# Instantiate partial product adder
|
|
for u in range(NUM_USRPS):
|
|
bee7fpga.sim_core.connect(bee7fpga.func_pp_comb[u], 0, bee7fpga.serdes_o[base_usrp_lane + u], 0)
|
|
|
|
# Coefficient consumer
|
|
bee7fpga.coeff_sink = rfnocsim.Consumer(bee7fpga.sim_core, bee7fpga.name + '/coeff_sink', 10e9/8, 0.0)
|
|
bee7fpga.sim_core.connect(bee7fpga.serdes_i[terminal_map['X'] + NUM_USRPS], 0, bee7fpga.coeff_sink, 0)
|
|
|
|
@classmethod
|
|
def connect(cls, sim_core, usrps, bee7blades, hosts, app_settings):
|
|
NUM_USRPS = 2
|
|
|
|
# Reshape BEE7s
|
|
# The blades are arranged in 3D Flattened Butterfly configuration
|
|
# with a dimension width of 4. The X and Z dimension represent row, col
|
|
# and the Y dimension represents the internal connections
|
|
bee7grid = []
|
|
for r in range(4):
|
|
bee7row = []
|
|
for c in range(4):
|
|
blade = bee7blades[4*r + c]
|
|
for f in range(blade.NUM_FPGAS):
|
|
cls.config_bitstream(blade.fpgas[f], app_settings, {'X':r, 'Y':f, 'Z':c})
|
|
bee7row.append(blade)
|
|
bee7grid.append(bee7row)
|
|
|
|
# USRP-Bee7 Connections
|
|
# Blades across the diagonal are connected to USRPs
|
|
for x in range(4):
|
|
for y in range(4):
|
|
for z in range(4):
|
|
for u in range(NUM_USRPS):
|
|
usrp_num = cls.get_radio_num({'X':x,'Y':y,'Z':z}, u, NUM_USRPS)
|
|
(router_map, terminal_map) = cls.get_portmap({'X':x,'Y':y,'Z':z})
|
|
sim_core.connect_bidir(
|
|
usrps[usrp_num], 0,
|
|
bee7grid[x][z], hw.Bee7Blade.io_lane(y, terminal_map['X'] + u), 'SAMP')
|
|
|
|
# Bee7-Bee7 Connections
|
|
null_srcs = []
|
|
for row in range(4):
|
|
for col in range(4):
|
|
for fpga in range(4):
|
|
(src_map, t) = cls.get_portmap({'X':row,'Y':fpga,'Z':col})
|
|
for dst in range(4):
|
|
if row != dst:
|
|
(dst_map, t) = cls.get_portmap({'X':dst,'Y':fpga,'Z':col})
|
|
sim_core.connect_multi(
|
|
bee7grid[row][col],
|
|
list(range(hw.Bee7Blade.io_lane(fpga, src_map['X'][dst]), hw.Bee7Blade.io_lane(fpga, src_map['X'][dst]+4))),
|
|
bee7grid[dst][col],
|
|
list(range(hw.Bee7Blade.io_lane(fpga, dst_map['X'][row]), hw.Bee7Blade.io_lane(fpga, dst_map['X'][row]+4))),
|
|
'SAMP')
|
|
if col != dst:
|
|
(dst_map, t) = cls.get_portmap({'X':row,'Y':fpga,'Z':dst})
|
|
sim_core.connect_multi(
|
|
bee7grid[row][col],
|
|
list(range(hw.Bee7Blade.io_lane(fpga, src_map['Z'][dst]), hw.Bee7Blade.io_lane(fpga, src_map['Z'][dst]+4))),
|
|
bee7grid[row][dst],
|
|
list(range(hw.Bee7Blade.io_lane(fpga, dst_map['Z'][col]), hw.Bee7Blade.io_lane(fpga, dst_map['Z'][col]+4))),
|
|
'PP', 'blue')
|
|
|
|
# Host connection
|
|
for row in range(4):
|
|
for col in range(4):
|
|
for fpga in range(4):
|
|
(router_map, terminal_map) = cls.get_portmap({'X':row,'Y':row,'Z':col})
|
|
sim_core.connect_bidir(
|
|
hosts[row], col*4 + fpga,
|
|
bee7grid[row][col], hw.Bee7Blade.io_lane(fpga, terminal_map['X'] + NUM_USRPS), 'COEFF', 'red')
|