Initial release of afu-directed support

.gitignore

@@ -1,14 +1,14 @@
 *.a
 *.so
 *.d
+*.d-e
 *.o
 *.sl
 .*.swp
 gmon.out
 debug.log
 common/misc
 debug/debug
-libcxl/include
 pslse/pslse
 test/afu/afu
 gprof.out

common/TestAFU_config.c

@@ -23,6 +23,15 @@
 #include <stdio.h>
 #include "TestAFU_config.h"
 
+// Zero out all machine config registers
+void init_machine(MachineConfig *machine)
+{
+	int i;
+
+	for (i = 0; i < 4; i++)
+		machine->config[i] = 0;
+}
+
 // Function to set most commonly used elements
 int config_machine(MachineConfig *machine, uint16_t context, uint16_t command, uint16_t command_size, uint16_t min_delay, uint16_t max_delay, uint64_t memory_base_address, uint64_t memory_size, uint8_t enable_always)
 {
@@ -47,22 +56,25 @@ int config_machine(MachineConfig *machine, uint16_t context, uint16_t command, u
 }
 
 // Helper function to calculate AFU machine register MMIO space offset
-static int _machine_base_address_index(uint16_t index)
+static int _machine_base_address_index(uint16_t index, int dedicated)
 {
 	int machine_config_base_address;
 	machine_config_base_address = index << 5;
-	machine_config_base_address = machine_config_base_address | 0x1000;
+	if (dedicated)
+		machine_config_base_address += 0x1000;
 	return machine_config_base_address;
 }
 
 // Function to write config to AFU MMIO space
-int enable_machine(struct cxl_afu_h *afu, MachineConfig *machine, uint16_t index)
+int enable_machine(struct cxl_afu_h *afu, MachineConfig *machine,
+		   uint16_t index, int dedicated)
 {
 	int i;
-	int machineConfig_baseaddress = _machine_base_address_index(index);
+	int machineConfig_baseaddress = _machine_base_address_index(index, dedicated);
 	for (i = 3; i >= 0; --i){
 		uint64_t data = machine->config[i];
-		if (cxl_mmio_write64(afu, machineConfig_baseaddress + (i * 8), data))
+		if (cxl_mmio_write64(afu, machineConfig_baseaddress + (i * 8),
+		    data))
 		{
 			printf("Failed to write data\n");
 			return -1;
@@ -73,12 +85,15 @@ int enable_machine(struct cxl_afu_h *afu, MachineConfig *machine, uint16_t index
 }
 
 // Function to read config from AFU
-int poll_machine(struct cxl_afu_h *afu, MachineConfig *machine, uint16_t index){
+int poll_machine(struct cxl_afu_h *afu, MachineConfig *machine, uint16_t index,
+		 int dedicated){
 	int i;
-	int machineConfig_baseaddress = _machine_base_address_index(index);
+	int machineConfig_baseaddress = _machine_base_address_index(index,
+								    dedicated);
 	for (i = 0; i < 2; ++i){
 		uint64_t temp;
-		if (cxl_mmio_read64(afu, machineConfig_baseaddress + (i * 8), &temp))
+		if (cxl_mmio_read64(afu, machineConfig_baseaddress + (i * 8),
+				    &temp))
 		{
 			printf("Failed to read data\n");
 			return -1;
@@ -90,38 +105,48 @@ int poll_machine(struct cxl_afu_h *afu, MachineConfig *machine, uint16_t index){
 }
 
 // Function to set most commonly used elements and write to AFU MMIO space
-int config_and_enable_machine(struct cxl_afu_h *afu, MachineConfig *machine, uint16_t mach_num, uint16_t context, uint16_t command, uint16_t command_size, uint16_t min_delay, uint16_t max_delay, uint64_t memory_base_address, uint64_t memory_size, uint8_t enable_always)
+int config_and_enable_machine(struct cxl_afu_h *afu, MachineConfig *machine, uint16_t mach_num, uint16_t context, uint16_t command, uint16_t command_size, uint16_t min_delay, uint16_t max_delay, uint64_t memory_base_address, uint64_t memory_size, uint8_t enable_always, int dedicated)
 {
 	if (config_machine(machine, context, command, command_size, min_delay,
 			   max_delay, memory_base_address, memory_size,
 			   enable_always))
 		return -1;
-	if (enable_machine(afu, machine, mach_num))
+	if (enable_machine(afu, machine, mach_num, dedicated))
 		return -1;
 	return 0;
 }
 
 // Wait for response from AFU machine
-int get_response(struct cxl_afu_h *afu, MachineConfig *machine, uint16_t mach_num)
+int get_response(struct cxl_afu_h *afu, MachineConfig *machine,
+		 uint16_t mach_num, int dedicated)
 {
 	uint8_t response;
 
 	do {
-		poll_machine(afu, machine, mach_num);
+		if (poll_machine(afu, machine, mach_num, dedicated) < 0)
+			return 0xFF;
 		get_machine_config_response_code(machine, &response);
 	} while (response == 0xFF);
 	return response;
 }
 
 // Function to set most commonly used elements, write to AFU MMIO space and
 // wait for command completion
-int config_enable_and_run_machine(struct cxl_afu_h *afu, MachineConfig *machine, uint16_t mach_num, uint16_t context, uint16_t command, uint16_t command_size, uint16_t min_delay, uint16_t max_delay, uint64_t memory_base_address, uint64_t memory_size)
+int config_enable_and_run_machine(struct cxl_afu_h *afu, MachineConfig *machine, uint16_t mach_num, uint16_t context, uint16_t command, uint16_t command_size, uint16_t min_delay, uint16_t max_delay, uint64_t memory_base_address, uint64_t memory_size, int dedicated)
 {
+	int rc;
+
 	if (config_and_enable_machine(afu, machine, mach_num, context, command,
 				      command_size, min_delay, max_delay,
-				      memory_base_address, memory_size, 0) < 0)
+				      memory_base_address, memory_size, 0,
+				      dedicated) < 0)
+		return -1;
+
+	rc = get_response(afu, machine, mach_num, dedicated);
+	if (rc==0xFF)
 		return -1;
-	return get_response(afu, machine, mach_num);
+
+	return rc;
 }
 
 //////////////////////////////////

common/TestAFU_config.h

@@ -24,31 +24,38 @@
 #include <inttypes.h>
 #include "libcxl.h"
 
+#define DEDICATED 1
+#define DIRECTED 0
+#define PPPSA_OFFSET 0x1000
+#define PPPSA_SIZE 0x1000
+
 // Strucure to configure AFU
 typedef struct AFUConfig
 {
 	uint64_t config[4];
 } MachineConfig;
 
+// Zero out all machine config registers
+void init_machine(MachineConfig *machine);
 
 // Function to set most commonly used elements
 int config_machine(MachineConfig *machine, uint16_t context, uint16_t command, uint16_t command_size, uint16_t min_delay, uint16_t max_delay, uint64_t memory_base_address, uint64_t memory_size, uint8_t enable_always);
 
 // Function to write config to AFU MMIO space
-int enable_machine(struct cxl_afu_h *afu, MachineConfig *machine, uint16_t index);
+int enable_machine(struct cxl_afu_h *afu, MachineConfig *machine, uint16_t index, int dedicated);
 
 // Function to set most commonly used elements and write to AFU MMIO space
-int config_and_enable_machine(struct cxl_afu_h *afu, MachineConfig *machine, uint16_t mach_num, uint16_t context, uint16_t command, uint16_t command_size, uint16_t min_delay, uint16_t max_delay, uint64_t memory_base_address, uint64_t memory_size, uint8_t enable_always);
+int config_and_enable_machine(struct cxl_afu_h *afu, MachineConfig *machine, uint16_t mach_num, uint16_t context, uint16_t command, uint16_t command_size, uint16_t min_delay, uint16_t max_delay, uint64_t memory_base_address, uint64_t memory_size, uint8_t enable_always, int dedicated);
 
 // Function to read config from AFU
-int poll_machine(struct cxl_afu_h *afu, MachineConfig *machine, uint16_t index);
+int poll_machine(struct cxl_afu_h *afu, MachineConfig *machine, uint16_t index, int dedicated);
 
 // Wait for response from AFU machine
-int get_response(struct cxl_afu_h *afu, MachineConfig *machine, uint16_t mach_num);
+int get_response(struct cxl_afu_h *afu, MachineConfig *machine, uint16_t mach_num, int dedicated);
 
 // Function to set most commonly used elements, write to AFU MMIO space and
 // wait for command completion
-int config_enable_and_run_machine(struct cxl_afu_h *afu, MachineConfig *machine, uint16_t mach_num, uint16_t context, uint16_t command, uint16_t command_size, uint16_t min_delay, uint16_t max_delay, uint64_t memory_base_address, uint64_t memory_size);
+int config_enable_and_run_machine(struct cxl_afu_h *afu, MachineConfig *machine, uint16_t mach_num, uint16_t context, uint16_t command, uint16_t command_size, uint16_t min_delay, uint16_t max_delay, uint64_t memory_base_address, uint64_t memory_size, int dedicated);
 
 // Enable always field is bits[0] of double-word 0
 void set_machine_config_enable_always(MachineConfig* machine);

common/psl_interface.c

@@ -52,7 +52,7 @@ static void set_protocol_level(struct AFU_EVENT *event, uint32_t primary,
 	    (event->proto_tertiary != tertiary)) {
 		printf
 		    ("PSL_SOCKET:WARNING: Adjusting PSL interface protocol level!\n");
-		printf("PSL_SOCKET:\tPlease review changes betwen levels.\n");
+		printf("PSL_SOCKET:\tPlease review changes between levels.\n");
 		printf("PSL_SOCKET:\tSupported PSL protocol level: %d.%d.%d\n",
 		       event->proto_primary, event->proto_secondary,
 		       event->proto_tertiary);
@@ -140,6 +140,17 @@ static int establish_protocol(struct AFU_EVENT *event)
 		tertiary += (uint32_t) byte;
 	}
 
+	// Check for broken levels
+	if ((primary == 0) && (secondary == 9908) && (tertiary == 0)) {
+		printf("Remote psl_interface code using broken code level!\n");
+		printf("\tLocal psl_interface level:%d.%d.%d\n",
+		       event->proto_primary, event->proto_secondary,
+		       event->proto_tertiary);
+		printf("\tRemote psl_interface level:%d.%d.%d\n",
+		       primary, secondary, tertiary);
+		return PSL_BAD_SOCKET;
+	}
+
 	// Test if other side with adjust protocol level down
 	if (primary > event->proto_primary)
 		return PSL_SUCCESS;
@@ -698,8 +709,8 @@ static int psl_signal_psl_model(struct AFU_EVENT *event)
 	if (event->command_valid) {
 		event->tbuf[0] = event->tbuf[0] | 0x01;
 		event->tbuf[bp++] = event->command_tag;
-		event->tbuf[bp++] = (((event->command_abort) << 4) & 0x70) |
-		    (((event->command_code) >> 8) & 0x0F);
+		event->tbuf[bp++] = (((event->command_abort) << 5) & 0xE0) |
+		    (((event->command_code) >> 8) & 0x1F);
 		event->tbuf[bp++] = event->command_code & 0xFF;
 		event->tbuf[bp++] =
 		    (((event->command_tag_parity) << 6) & 0x40) |
@@ -841,8 +852,8 @@ int psl_get_afu_events(struct AFU_EVENT *event)
 	if ((event->rbuf[0] & 0x01) != 0) {
 		event->command_valid = 1;
 		event->command_tag = event->rbuf[rbc++];
-		event->command_abort = (event->rbuf[rbc] >> 4) & 0x7;
-		event->command_code = (event->rbuf[rbc++] & 0x0F) << 8;
+		event->command_abort = (event->rbuf[rbc] >> 5) & 0x7;
+		event->command_code = (event->rbuf[rbc++] & 0x1F) << 8;
 		event->command_code = event->command_code | event->rbuf[rbc++];
 		event->command_tag_parity = (event->rbuf[rbc] >> 6) & 0x01;
 		event->command_code_parity = (event->rbuf[rbc] >> 5) & 0x01;

common/psl_interface_t.h

@@ -29,7 +29,7 @@
 #define PSL_BUFFER_SIZE 200
 #define PROTOCOL_PRIMARY 0
 #define PROTOCOL_SECONDARY 9908
-#define PROTOCOL_TERTIARY 0
+#define PROTOCOL_TERTIARY 1
 
 /* Return codes for interface functions */
 
@@ -68,6 +68,7 @@
 #define PSL_LLCMD_TERMINATE 0x0001000000000000LL
 #define PSL_LLCMD_REMOVE 0x0002000000000000LL
 #define PSL_LLCMD_ADD 0x0005000000000000LL
+#define PSL_LLCMD_CONTEXT_MASK 0x000000000000FFFFLL
 
 /* Response codes for PSL responses */
 

common/utils.c

@@ -185,14 +185,13 @@ int get_bytes_silent(int fd, int size, uint8_t * data, int timeout, int *abort)
 			break;
 		}
 		if (rc < 0) {
-			perror("bytes_ready");
+			warn_msg("bytes_ready:Socket disconnect");
 			break;
 		}
 
 		bytes = recv(fd, data, size - bytes, MSG_PEEK | MSG_DONTWAIT);
-		if ((bytes <= 0) && (errno != EINTR)) {
-			perror("recv");
-			warn_msg("Socket disconnect on recv");
+		if (((bytes < 0) && (errno != EINTR)) || !bytes) {
+			warn_msg("get_bytes_silent:Socket disconnect on recv");
 			return -1;
 		}
 	}

libcxl/Makefile

@@ -5,7 +5,8 @@ include Makefile.rules
 
 OBJS = libcxl.o debug.o utils.o
 
-all: $(COMMON_DIR)/misc/cxl.h libcxl.so libcxl.a
+#all: $(COMMON_DIR)/misc/cxl.h libcxl.so libcxl.a
+all: $(COMMON_DIR)/misc/cxl.h libcxl.a
 
 CHECK_HEADER = $(shell echo \\\#include\ $(1) | $(CC) $(CFLAGS) -E - > /dev/null 2>&1 && echo y || echo n)
 
@@ -14,8 +15,8 @@ ifeq ($(call CHECK_HEADER,"<misc/cxl.h>"),n)
 	$(call Q,CURL $(COMMON_DIR)/misc/cxl.h, mkdir $(COMMON_DIR)/misc 2>/dev/null; curl -o $(COMMON_DIR)/misc/cxl.h -s http://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/plain/include/uapi/misc/cxl.h)
 endif
 
-libcxl.so: $(OBJS)
-	$(call Q,CC, $(CC) $(CFLAGS) -shared $^ -o $@, $@) -Wl,--version-script symver.map
+#libcxl.so: $(OBJS)
+#	$(call Q,CC, $(CC) $(CFLAGS) -shared $^ -o $@, $@) -Wl,--version-script symver.map
 
 libcxl.a: $(OBJS)
 	$(call Q,AR, $(AR) rcs $@ $^, $@)

libcxl/Makefile.vars

@@ -5,7 +5,7 @@ AS = $(CROSS_COMPILE)as
 AR = $(CROSS_COMPILE)ar
 LD = $(CROSS_COMPILE)ld
 CC = $(CROSS_COMPILE)gcc
-CFLAGS += -Wall -Wunused-but-set-variable -I$(CURDIR) -I$(COMMON_DIR) -fPIC
+CFLAGS += -Wall -I$(CURDIR) -I$(COMMON_DIR) -fPIC
 
 ifeq ($(BIT32),y)
   CFLAGS += -m32