PrincetonUniversity · dpetrisko · May 23, 2023 · Jun 18, 2023 · Jun 18, 2023 · Jun 18, 2023
diff --git a/.gitmodules b/.gitmodules
@@ -7,3 +7,9 @@
 [submodule "piton/design/chipset/rv64_platform/bootrom/u-boot/uboot"]
 	path = piton/design/chipset/rv64_platform/bootrom/u-boot/uboot
 	url = git@github.com:u-boot/u-boot.git
+[submodule "piton/design/chip/tile/blackparrot/black-parrot"]
+    path = piton/design/chip/tile/blackparrot/black-parrot
+    url = https://github.com/black-parrot/black-parrot.git
+[submodule "piton/design/chip/tile/blackparrot/black-parrot-subsystems"]
+	path = piton/design/chip/tile/blackparrot/black-parrot-subsystems
+	url = https://github.com/black-parrot-hdk/black-parrot-subsystems
diff --git a/README.md b/README.md
@@ -129,7 +129,6 @@ Step 3. will then download and compile the RISC-V toolchain, the assembly tests
 
 > You will need Vivado 2018.2 or newer to build an FPGA bitstream with Ariane.
 
-
 #### Running RISC-V Tests and Benchmarks
 
 The RISC-V benchmarks are precompiled in the tool setup step mentioned above. You can run individual benchmarks by first building the simulation model with
@@ -505,3 +504,17 @@ The command will tell print the afi and agfi of your image. You can track the sy
 
 8. After the synthesis is done - you can go load it in your F1 instance!
 
+# Support for the BlackParrot RV64GC Core
+
+This version of OpenPiton supports the [64bit BlackParrot RISC-V processor](https://github.com/black-parrot/black-parrot) from University of Washington and Boston University. BlackParrot has been equipped with the P-MESH Cache Engine (PCE), which connects the BlackParrot Cache Engine Interface to the L1.5 cache provided by OpenPiton's P-Mesh. Instructions to run are identical to running with Ariane, except to use the -blackparrot flag instead.
+
+#### Planned Improvements
+
+The following items are currently under development and will be released soon.
+
+- Thorough validation of cache coherence.
+- RISC-V FESVR support in simulation.
+- Synthesis flow for large FPGAs.
+- Performance enhancements (cache re-parameterization, write-buffer throughput).
+
+Stay tuned!
diff --git a/piton/blackparrot_settings.bash b/piton/blackparrot_settings.bash
@@ -0,0 +1,12 @@
+export BLACKPARROT_ROOT=$DV_ROOT/design/chip/tile/blackparrot
+export BLACKPARROT_RTL_DIR=$BLACKPARROT_ROOT/black-parrot
+export BLACKPARROT_SUBSYSTEMS_DIR=$BLACKPARROT_ROOT/black-parrot-subsystems
+export BP_COMMON_DIR=$BLACKPARROT_RTL_DIR/bp_common
+export BP_TOP_DIR=$BLACKPARROT_RTL_DIR/bp_top
+export BP_FE_DIR=$BLACKPARROT_RTL_DIR/bp_fe
+export BP_BE_DIR=$BLACKPARROT_RTL_DIR/bp_be
+export BP_ME_DIR=$BLACKPARROT_RTL_DIR/bp_me
+export BP_EXTERNAL_DIR=$BLACKPARROT_RTL_DIR/external
+export BASEJUMP_STL_DIR=$BP_EXTERNAL_DIR/basejump_stl
+export HARDFLOAT_DIR=$BP_EXTERNAL_DIR/HardFloat
+
diff --git a/piton/design/chip/rtl/chip.v.pyv b/piton/design/chip/rtl/chip.v.pyv
@@ -1137,6 +1137,8 @@ module chip(
                 currenttype = "`PICORV32_TILE"
             elif (PITON_ARIANE):
                 currenttype = "`ARIANE_RV64_TILE"
+            elif (PITON_BLACKPARROT):
+                currenttype = "`BLACKPARROT_RV64_TILE"
             else:
                 currenttype = "`SPARC_TILE"
 

diff --git a/piton/design/chip/tile/ariane b/piton/design/chip/tile/ariane
diff --git a/piton/design/chip/tile/blackparrot/Flist.blackparrot b/piton/design/chip/tile/blackparrot/Flist.blackparrot
@@ -0,0 +1,4 @@
+$BLACKPARROT_ROOT/rtl/bp_common_piton_pkg.sv
++incdir+$BLACKPARROT_SUBSYSTEMS_DIR/openpiton/v/
+$BLACKPARROT_SUBSYSTEMS_DIR/openpiton/v/bp_pce.sv
+$BLACKPARROT_SUBSYSTEMS_DIR/openpiton/v/bp_piton_tile.sv
diff --git a/piton/design/chip/tile/blackparrot/black-parrot b/piton/design/chip/tile/blackparrot/black-parrot
diff --git a/piton/design/chip/tile/blackparrot/black-parrot-subsystems b/piton/design/chip/tile/blackparrot/black-parrot-subsystems
diff --git a/piton/design/chip/tile/blackparrot/rtl/bp_common_piton_pkg.sv b/piton/design/chip/tile/blackparrot/rtl/bp_common_piton_pkg.sv
@@ -0,0 +1,83 @@
+/*
+ * bp_common_test_pkg.sv
+ *
+ * This package contains extra testing configs which are not intended to be
+ *   synthesized or used in production. However, they are useful for testing.
+ *   This file can also be used as a template for 3rd parties wishing to
+ *   synthesize extra configs without modifying the BP source directly.
+ *
+ */
+
+  `include "bp_common_defines.svh"
+
+package bp_common_pkg;
+
+  `include "bp_common_accelerator_pkgdef.svh"
+  `include "bp_common_addr_pkgdef.svh"
+  //`include "bp_common_aviary_pkgdef.svh"
+  `include "bp_common_aviary_cfg_pkgdef.svh"
+
+  // Default configuration is unicore
+  localparam bp_proc_param_s bp_unicore_cfg_p = bp_default_cfg_p;
+
+  localparam bp_proc_param_s bp_unicore_parrotpiton_override_p =
+    '{cc_x_dim              : 16 // Maximum number of BlackParrot cores, increase if more are needed
+      ,cc_y_dim             : 1
+      ,ic_y_dim             : 0
+      ,icache_features      : (1 << e_cfg_enabled) | (1 << e_cfg_misaligned) | (1 << e_cfg_coherent)
+      ,icache_sets          : 128
+      ,icache_assoc         : 4
+      ,icache_block_width   : 256
+      ,icache_fill_width    : 256
+      ,dcache_features      : (1 << e_cfg_enabled) | (1 << e_cfg_coherent) | (1 << e_cfg_hit_under_miss)
+      ,dcache_sets          : 256
+      ,dcache_assoc         : 2
+      ,dcache_block_width   : 128
+      ,dcache_fill_width    : 128
+      // Actually L1.5 features
+      ,l2_features          : (1 << e_cfg_lr_sc)
+                              | (1 << e_cfg_amo_swap)
+                              | (1 << e_cfg_amo_fetch_logic)
+                              | (1 << e_cfg_amo_fetch_arithmetic)
+      ,default : "inv"
+      };
+  `bp_aviary_derive_cfg(bp_unicore_parrotpiton_cfg_p
+                        ,bp_unicore_parrotpiton_override_p
+                        ,bp_unicore_cfg_p
+                        );
+
+  parameter bp_proc_param_s [max_cfgs-1:0] all_cfgs_gp =
+  {
+    // Various testing configs
+    bp_unicore_parrotpiton_cfg_p 
+    ,bp_unicore_cfg_p
+
+    // A custom BP configuration generated from Makefile
+    ,bp_custom_cfg_p
+    // The default BP
+    ,bp_default_cfg_p
+  };
+
+  // This enum MUST be kept up to date with the parameter array above
+  typedef enum bit [lg_max_cfgs-1:0]
+  {
+    e_bp_unicore_parrotpiton_cfg                    = 3
+    ,e_bp_unicore_cfg                               = 2
+
+    // A custom BP configuration generated from `defines
+    ,e_bp_custom_cfg                                = 1
+    // The default BP
+    ,e_bp_default_cfg                               = 0
+  } bp_params_e;
+
+  `include "bp_common_bedrock_pkgdef.svh"
+  `include "bp_common_cache_pkgdef.svh"
+  `include "bp_common_cache_engine_pkgdef.svh"
+  `include "bp_common_cfg_bus_pkgdef.svh"
+  `include "bp_common_clint_pkgdef.svh"
+  `include "bp_common_core_pkgdef.svh"
+  `include "bp_common_host_pkgdef.svh"
+  `include "bp_common_rv64_pkgdef.svh"
+
+endpackage
+
diff --git a/piton/design/chip/tile/blackparrot/rtl/bsg_popcount_patched.sv b/piton/design/chip/tile/blackparrot/rtl/bsg_popcount_patched.sv
@@ -0,0 +1,79 @@
+`include "bsg_defines.sv"
+
+// MBT popcount
+//
+// 10-24-14
+//
+
+//module bsg_popcount #(parameter `BSG_INV_PARAM(width_p))
+module bsg_popcount #(parameter width_p=1)
+   (input [width_p-1:0] i
+    , output [$clog2(width_p+1)-1:0] o
+    );
+
+   // perf fixme: better to round up to nearest power of two and then
+   // recurse with side full and one side minimal
+   //
+   // e.g-> 80 -> 128/2 = 64 --> (64,16)
+   //
+   // possibly slightly better is to use 2^N-1:
+   // 
+   // for items that are 5..7 bits wide, we make sure to
+   // split into a 4 and a 1/2/3; since the four is relatively optimized.
+   //
+
+   localparam first_half_lp  = `BSG_MAX(4,width_p - (width_p >> 1));
+   localparam second_half_lp = width_p - first_half_lp;
+
+   if (width_p <= 3)
+     begin : lt3
+        assign o[0] = ^i;
+
+        if (width_p == 2)
+          assign o[1] = & i;
+        else
+          if (width_p == 3)
+            assign o[1] = (&i[1:0]) | (&i[2:1]) | (i[0]&i[2]);
+     end
+   else
+     // http://www.wseas.us/e-library/conferences/2006hangzhou/papers/531-262.pdf
+
+     if (width_p == 4)
+       begin : four
+          // half adders
+          wire [1:0] s0 = { ^i[3:2], ^i[1:0]};
+          wire [1:0] c0 = { &i[3:2], &i[1:0]};
+
+          // low bit is xor of all bits
+          assign o[0] =  ^s0;
+
+          // middle bit is: ab ^ cd
+          //            or  (a^b) & (c^d)
+
+          assign o[1] =  (^c0) | (&s0);
+
+          // high bit is and of all bits
+
+          assign o[2] =  &c0;
+       end
+     else
+       begin : recurse
+          wire [$clog2(first_half_lp+1)-1:0]  lo;
+          wire [$clog2(second_half_lp+1)-1:0] hi;
+
+          bsg_popcount #(.width_p(first_half_lp))
+             left(.i(i[0+:first_half_lp])
+                  ,.o(lo)
+                  );
+
+          bsg_popcount #(.width_p(second_half_lp))
+          right(.i(i[first_half_lp+:second_half_lp])
+                ,.o(hi)
+                );
+
+          assign o = lo+hi;
+       end
+
+endmodule // bsg_popcount
+
+`BSG_ABSTRACT_MODULE(bsg_popcount)
diff --git a/piton/design/chip/tile/l2/rtl/l2_pipe1.v b/piton/design/chip/tile/l2/rtl/l2_pipe1.v
@@ -367,6 +367,7 @@ wire stall_smc_buf_S4;
 wire msg_from_mshr_S4;
 wire req_recycle_S4;
 wire inv_fwd_pending_S4;
+wire msg_data_8B_amo_S4;
 
 wire msg_send_valid;
 wire msg_send_ready;
@@ -695,6 +696,7 @@ l2_pipe1_ctrl ctrl(
     .msg_from_mshr_S4           (msg_from_mshr_S4),
     .req_recycle_S4             (req_recycle_S4),
     .inv_fwd_pending_S4         (inv_fwd_pending_S4),
+    .msg_data_8B_amo_S4         (msg_data_8B_amo_S4),
     .dir_sharer_S4              (dir_sharer_S4),
     .dir_sharer_counter_S4      (dir_sharer_counter_S4),
     .cas_cmp_en_S4              (cas_cmp_en_S4),
@@ -862,6 +864,7 @@ l2_pipe1_dpath dpath(
     .msg_from_mshr_S4           (msg_from_mshr_S4),
     .req_recycle_S4             (req_recycle_S4),
     .inv_fwd_pending_S4         (inv_fwd_pending_S4),
+    .msg_data_8B_amo_S4         (msg_data_8B_amo_S4),
     .cas_cmp_en_S4              (cas_cmp_en_S4),
     .atomic_read_data_en_S4     (atomic_read_data_en_S4),
     .cas_cmp_data_size_S4       (cas_cmp_data_size_S4),

diff --git a/piton/design/chip/tile/l2/rtl/l2_pipe1_ctrl.v.pyv b/piton/design/chip/tile/l2/rtl/l2_pipe1_ctrl.v.pyv
@@ -281,6 +281,7 @@ module l2_pipe1_ctrl(
     output reg  msg_from_mshr_S4,
     output reg req_recycle_S4,
     output reg inv_fwd_pending_S4,
+    output reg msg_data_8B_amo_S4,
 
     output wire [`L2_OWNER_BITS-1:0] dir_sharer_S4,
     output reg [`L2_OWNER_BITS-1:0] dir_sharer_counter_S4,
@@ -369,7 +370,9 @@ reg [`L2_ADDR_TYPE_WIDTH-1:0] addr_type_S1;
 reg [`L2_ADDR_OP_WIDTH-1:0] addr_op_S1;
 reg special_addr_type_S1;
 reg msg_data_rd_S1;
+reg msg_amo_S1;
 reg msg_data_16B_amo_S1;
+reg msg_data_8B_amo_S1;
 
 always @ *
 begin
@@ -883,8 +886,24 @@ end
 
 always @ *
 begin
-    msg_data_16B_amo_S1 = valid_S1 && ((msg_type_trans_S1 == `MSG_TYPE_SWAP_P1_REQ) || (msg_type_trans_S1 == `MSG_TYPE_SWAP_P2_REQ) || (msg_type_trans_S1 == `MSG_TYPE_SWAPWB_P1_REQ) || (msg_type_trans_S1 == `MSG_TYPE_SWAPWB_P2_REQ)) && (data_size_S1 == `MSG_DATA_SIZE_16B);
+    msg_amo_S1 = valid_S1
+              && (msg_type_trans_S1 == `MSG_TYPE_AMO_ADD_P1_REQ || msg_type_trans_S1 == `MSG_TYPE_AMO_ADD_P2_REQ
+              ||  msg_type_trans_S1 == `MSG_TYPE_AMO_AND_P1_REQ || msg_type_trans_S1 == `MSG_TYPE_AMO_AND_P2_REQ
+              ||  msg_type_trans_S1 == `MSG_TYPE_AMO_OR_P1_REQ || msg_type_trans_S1 == `MSG_TYPE_AMO_OR_P2_REQ
+              ||  msg_type_trans_S1 == `MSG_TYPE_AMO_XOR_P1_REQ || msg_type_trans_S1 == `MSG_TYPE_AMO_XOR_P2_REQ
+              ||  msg_type_trans_S1 == `MSG_TYPE_AMO_MAX_P1_REQ || msg_type_trans_S1 == `MSG_TYPE_AMO_MAX_P2_REQ
+              ||  msg_type_trans_S1 == `MSG_TYPE_AMO_MAXU_P1_REQ || msg_type_trans_S1 == `MSG_TYPE_AMO_MAXU_P2_REQ
+              ||  msg_type_trans_S1 == `MSG_TYPE_AMO_MIN_P1_REQ || msg_type_trans_S1 == `MSG_TYPE_AMO_MIN_P2_REQ
+              ||  msg_type_trans_S1 == `MSG_TYPE_AMO_MINU_P1_REQ || msg_type_trans_S1 == `MSG_TYPE_AMO_MINU_P2_REQ
+              ||  msg_type_trans_S1 == `MSG_TYPE_SWAP_P1_REQ || msg_type_trans_S1 == `MSG_TYPE_SWAP_P2_REQ
+              ||  msg_type_trans_S1 == `MSG_TYPE_SWAPWB_P1_REQ || msg_type_trans_S1 == `MSG_TYPE_SWAPWB_P2_REQ);
+end
+
+always @ *
+begin
+    msg_data_16B_amo_S1 = valid_S1 && msg_amo_S1 && (data_size_S1 == `MSG_DATA_SIZE_16B);
     //msg_data_16B_amo_masked_S1 = valid_S1 && ((msg_type_trans_S1 == `MSG_TYPE_SWAPWB_P1_REQ) || (msg_type_trans_S1 == `MSG_TYPE_SWAPWB_P2_REQ)) && (data_size_S1 == `MSG_DATA_SIZE_16B);
+    msg_data_8B_amo_S1 = valid_S1 && msg_amo_S1 && (data_size_S1 == `MSG_DATA_SIZE_8B);
 end
 
 always @ *
@@ -1056,6 +1075,7 @@ reg mshr_smc_miss_S2_f;
 reg [`L2_MSHR_INDEX_WIDTH-1:0] mshr_pending_index_S2_f;
 reg special_addr_type_S2_f;
 reg msg_data_rd_S2_f;
+reg msg_data_8B_amo_S2_f;
 
 always @ (posedge clk)
 begin
@@ -1074,6 +1094,7 @@ begin
         special_addr_type_S2_f <= 0;
         msg_data_rd_S2_f <= 0;
         msg_data_16B_amo_S2_f <= 1'b0;
+        msg_data_8B_amo_S2_f <= 1'b0;
         amo_alu_op_S2_f <= `L2_AMO_ALU_OP_WIDTH'b0;
     end
     else if (!stall_S2)
@@ -1095,6 +1116,7 @@ begin
         special_addr_type_S2_f <= special_addr_type_S1;
         msg_data_rd_S2_f <= msg_data_rd_S1;
         msg_data_16B_amo_S2_f <= msg_data_16B_amo_S1;
+        msg_data_8B_amo_S2_f <= msg_data_8B_amo_S1;
         amo_alu_op_S2_f <= amo_alu_op_S1;
     end
 end
@@ -2443,6 +2465,7 @@ reg [`L2_MSHR_STATE_BITS-1:0] mshr_state_in_S3_f;
 reg [`L2_MSHR_INDEX_WIDTH-1:0] mshr_pending_index_S3_f;
 reg special_addr_type_S3_f;
 reg req_recycle_S3_f;
+reg msg_data_8B_amo_S3_f;
 
 always @ (posedge clk)
 begin
@@ -2466,6 +2489,7 @@ begin
         mshr_pending_index_S3_f <= 0;
         special_addr_type_S3_f <= 0;
         req_recycle_S3_f <= 0;
+        msg_data_8B_amo_S3_f <= 0;
     end
     else if (!stall_S3)
     begin
@@ -2487,6 +2511,7 @@ begin
         mshr_pending_index_S3_f <= mshr_pending_index_S2_f;
         special_addr_type_S3_f <= special_addr_type_S2_f;
         req_recycle_S3_f <= req_recycle_S2;
+        msg_data_8B_amo_S3_f <= msg_data_8B_amo_S2_f;
     end
 end
 
@@ -2624,6 +2649,7 @@ reg [`L2_MSHR_INDEX_WIDTH-1:0] mshr_pending_index_S4_f;
 reg special_addr_type_S4_f;
 reg [`L2_DIR_ARRAY_WIDTH-1:0] dir_data_S4_f;
 reg req_recycle_S4_f;
+reg msg_data_8B_amo_S4_f;
 
 
 always @ (posedge clk)
@@ -2649,6 +2675,7 @@ begin
         special_addr_type_S4_f <= 0;
         dir_data_S4_f <= 0;
         req_recycle_S4_f <= 0;
+        msg_data_8B_amo_S4_f <= 0;
     end
     else if (!stall_S4)
     begin
@@ -2671,6 +2698,7 @@ begin
         special_addr_type_S4_f <= special_addr_type_S3_f;
         dir_data_S4_f <= dir_data_S3;
         req_recycle_S4_f <= req_recycle_S3;
+        msg_data_8B_amo_S4_f <= msg_data_8B_amo_S3_f;
     end
 end
 
@@ -2722,6 +2750,7 @@ begin
     special_addr_type_S4 = special_addr_type_S4_f;
     dir_data_S4 = dir_data_S4_f;
     msg_from_mshr_S4 = msg_from_mshr_S4_f;
+    msg_data_8B_amo_S4 = msg_data_8B_amo_S4_f;
 end