diff --git a/platform/broadcom/sonic-platform-modules-cel/debian/platform-modules-questone2bd.init b/platform/broadcom/sonic-platform-modules-cel/debian/platform-modules-questone2bd.init index a7ed2957814e..2f61f33c928d 100644 --- a/platform/broadcom/sonic-platform-modules-cel/debian/platform-modules-questone2bd.init +++ b/platform/broadcom/sonic-platform-modules-cel/debian/platform-modules-questone2bd.init @@ -20,6 +20,7 @@ start) modprobe pmbus_core modprobe i2c-imc allow_unsafe_access=1 modprobe baseboard_cpld + modprobe i2c-ocores-polling modprobe switchboard_fpga allow_unsafe_i2c_access=1 modprobe mc24lc64t modprobe i2c_dev_sysfs @@ -51,6 +52,9 @@ start) echo lm75b 0x48 > /sys/bus/i2c/devices/i2c-9/new_device echo lm75b 0x4d > /sys/bus/i2c/devices/i2c-67/new_device + # Populate voltage sensors reading + echo max11617 0x35 > /sys/bus/i2c/devices/i2c-71/new_device + # Populate EEPROM ## FAN echo 24lc64t 0x50 > /sys/bus/i2c/devices/i2c-2/new_device diff --git a/platform/broadcom/sonic-platform-modules-cel/questone2bd/modules/Makefile b/platform/broadcom/sonic-platform-modules-cel/questone2bd/modules/Makefile index 60a70645f18e..86b2e9272a5b 100644 --- a/platform/broadcom/sonic-platform-modules-cel/questone2bd/modules/Makefile +++ b/platform/broadcom/sonic-platform-modules-cel/questone2bd/modules/Makefile @@ -1,2 +1,2 @@ -obj-m := mc24lc64t.o baseboard_cpld.o switchboard_fpga.o i2c-imc.o +obj-m := mc24lc64t.o baseboard_cpld.o switchboard_fpga.o i2c-imc.o i2c-ocores-polling.o obj-m += i2c_dev_sysfs.o syscpld.o fancpld.o dps1100.o diff --git a/platform/broadcom/sonic-platform-modules-cel/questone2bd/modules/i2c-ocores-polling.c b/platform/broadcom/sonic-platform-modules-cel/questone2bd/modules/i2c-ocores-polling.c new file mode 100644 index 000000000000..b054849a0dd6 --- /dev/null +++ b/platform/broadcom/sonic-platform-modules-cel/questone2bd/modules/i2c-ocores-polling.c @@ -0,0 +1,794 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * i2c-ocores-polling.c: I2C bus driver for OpenCores I2C controller + * (https://opencores.org/project/i2c/overview) + * + * Peter Korsgaard + * + * Support for the GRLIB port of the controller by + * Andreas Larsson + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "i2c-ocores-polling.h" + +/* + * 'process_lock' exists because ocores_process() and ocores_process_timeout() + * can't run in parallel. + */ +struct ocores_i2c { + void __iomem *base; + int iobase; + u32 reg_shift; + u32 reg_io_width; + wait_queue_head_t wait; + struct i2c_adapter adap; + struct i2c_msg *msg; + int adapter_nr; + int pos; + int nmsgs; + int state; /* see STATE_ */ + spinlock_t process_lock; + struct clk *clk; + int ip_clock_khz; + int bus_clock_khz; + void (*setreg)(struct ocores_i2c *i2c, int reg, u8 value); + u8 (*getreg)(struct ocores_i2c *i2c, int reg); +}; + +/* registers */ +#define OCI2C_PRELOW 0 +#define OCI2C_PREHIGH 1 +#define OCI2C_CONTROL 2 +#define OCI2C_DATA 3 +#define OCI2C_CMD 4 /* write only */ +#define OCI2C_STATUS 4 /* read only, same address as OCI2C_CMD */ + +#define OCI2C_CTRL_IEN 0x40 +#define OCI2C_CTRL_EN 0x80 + +#define OCI2C_CMD_START 0x91 +#define OCI2C_CMD_STOP 0x41 +#define OCI2C_CMD_READ 0x21 +#define OCI2C_CMD_WRITE 0x11 +#define OCI2C_CMD_READ_ACK 0x21 +#define OCI2C_CMD_READ_NACK 0x29 +#define OCI2C_CMD_IACK 0x01 + +#define OCI2C_STAT_IF 0x01 +#define OCI2C_STAT_TIP 0x02 +#define OCI2C_STAT_ARBLOST 0x20 +#define OCI2C_STAT_BUSY 0x40 +#define OCI2C_STAT_NACK 0x80 + +#define STATE_DONE 0 +#define STATE_START 1 +#define STATE_WRITE 2 +#define STATE_READ 3 +#define STATE_ERROR 4 + +#define TYPE_OCORES 0 +#define TYPE_GRLIB 1 + +static void oc_setreg_8(struct ocores_i2c *i2c, int reg, u8 value) +{ + iowrite8(value, i2c->base + (reg << i2c->reg_shift)); +} + +static void oc_setreg_16(struct ocores_i2c *i2c, int reg, u8 value) +{ + iowrite16(value, i2c->base + (reg << i2c->reg_shift)); +} + +static void oc_setreg_32(struct ocores_i2c *i2c, int reg, u8 value) +{ + iowrite32(value, i2c->base + (reg << i2c->reg_shift)); +} + +static void oc_setreg_16be(struct ocores_i2c *i2c, int reg, u8 value) +{ + iowrite16be(value, i2c->base + (reg << i2c->reg_shift)); +} + +static void oc_setreg_32be(struct ocores_i2c *i2c, int reg, u8 value) +{ + iowrite32be(value, i2c->base + (reg << i2c->reg_shift)); +} + +static inline u8 oc_getreg_8(struct ocores_i2c *i2c, int reg) +{ + return ioread8(i2c->base + (reg << i2c->reg_shift)); +} + +static inline u8 oc_getreg_16(struct ocores_i2c *i2c, int reg) +{ + return ioread16(i2c->base + (reg << i2c->reg_shift)); +} + +static inline u8 oc_getreg_32(struct ocores_i2c *i2c, int reg) +{ + return ioread32(i2c->base + (reg << i2c->reg_shift)); +} + +static inline u8 oc_getreg_16be(struct ocores_i2c *i2c, int reg) +{ + return ioread16be(i2c->base + (reg << i2c->reg_shift)); +} + +static inline u8 oc_getreg_32be(struct ocores_i2c *i2c, int reg) +{ + return ioread32be(i2c->base + (reg << i2c->reg_shift)); +} + +static void oc_setreg_io_8(struct ocores_i2c *i2c, int reg, u8 value) +{ + outb(value, i2c->iobase + reg); +} + +static inline u8 oc_getreg_io_8(struct ocores_i2c *i2c, int reg) +{ + return inb(i2c->iobase + reg); +} + +static inline void oc_setreg(struct ocores_i2c *i2c, int reg, u8 value) +{ + i2c->setreg(i2c, reg, value); +} + +static inline u8 oc_getreg(struct ocores_i2c *i2c, int reg) +{ + return i2c->getreg(i2c, reg); +} + +static void ocores_process(struct ocores_i2c *i2c, u8 stat) +{ + struct i2c_msg *msg = i2c->msg; + unsigned long flags; + + /* + * If we spin here is because we are in timeout, so we are going + * to be in STATE_ERROR. See ocores_process_timeout() + */ + spin_lock_irqsave(&i2c->process_lock, flags); + + if ((i2c->state == STATE_DONE) || (i2c->state == STATE_ERROR)) { + /* stop has been sent */ + oc_setreg(i2c, OCI2C_CMD, OCI2C_CMD_IACK); + wake_up(&i2c->wait); + goto out; + } + + /* error? */ + if (stat & OCI2C_STAT_ARBLOST) { + i2c->state = STATE_ERROR; + oc_setreg(i2c, OCI2C_CMD, OCI2C_CMD_STOP); + goto out; + } + + if ((i2c->state == STATE_START) || (i2c->state == STATE_WRITE)) { + i2c->state = + (msg->flags & I2C_M_RD) ? STATE_READ : STATE_WRITE; + + if (stat & OCI2C_STAT_NACK) { + i2c->state = STATE_ERROR; + oc_setreg(i2c, OCI2C_CMD, OCI2C_CMD_STOP); + goto out; + } + } else { + msg->buf[i2c->pos++] = oc_getreg(i2c, OCI2C_DATA); + } + + /* end of msg? */ + if (i2c->pos == msg->len) { + i2c->nmsgs--; + i2c->msg++; + i2c->pos = 0; + msg = i2c->msg; + + if (i2c->nmsgs) { /* end? */ + /* send start? */ + if (!(msg->flags & I2C_M_NOSTART)) { + u8 addr = i2c_8bit_addr_from_msg(msg); + + i2c->state = STATE_START; + + oc_setreg(i2c, OCI2C_DATA, addr); + oc_setreg(i2c, OCI2C_CMD, OCI2C_CMD_START); + goto out; + } + i2c->state = (msg->flags & I2C_M_RD) + ? STATE_READ : STATE_WRITE; + } else { + i2c->state = STATE_DONE; + oc_setreg(i2c, OCI2C_CMD, OCI2C_CMD_STOP); + goto out; + } + } + + if (i2c->state == STATE_READ) { + oc_setreg(i2c, OCI2C_CMD, i2c->pos == (msg->len-1) ? + OCI2C_CMD_READ_NACK : OCI2C_CMD_READ_ACK); + } else { + oc_setreg(i2c, OCI2C_DATA, msg->buf[i2c->pos++]); + oc_setreg(i2c, OCI2C_CMD, OCI2C_CMD_WRITE); + } + +out: + spin_unlock_irqrestore(&i2c->process_lock, flags); +} + +static irqreturn_t ocores_isr(int irq, void *dev_id) +{ + struct ocores_i2c *i2c = dev_id; + u8 stat = oc_getreg(i2c, OCI2C_STATUS); + + if (!(stat & OCI2C_STAT_IF)) + return IRQ_NONE; + + ocores_process(i2c, stat); + + return IRQ_HANDLED; +} + +/** + * Process timeout event + * @i2c: ocores I2C device instance + */ +static void ocores_process_timeout(struct ocores_i2c *i2c) +{ + unsigned long flags; + + spin_lock_irqsave(&i2c->process_lock, flags); + i2c->state = STATE_ERROR; + oc_setreg(i2c, OCI2C_CMD, OCI2C_CMD_STOP); + spin_unlock_irqrestore(&i2c->process_lock, flags); +} + +/** + * Wait until something change in a given register + * @i2c: ocores I2C device instance + * @reg: register to query + * @mask: bitmask to apply on register value + * @val: expected result + * @timeout: timeout in jiffies + * + * Timeout is necessary to avoid to stay here forever when the chip + * does not answer correctly. + * + * Return: 0 on success, -ETIMEDOUT on timeout + */ +static int ocores_wait(struct ocores_i2c *i2c, + int reg, u8 mask, u8 val, + const unsigned long timeout) +{ + unsigned long j; + + j = jiffies + timeout; + while (1) { + u8 status = oc_getreg(i2c, reg); + + if ((status & mask) == val) + break; + + if (time_after(jiffies, j)) + return -ETIMEDOUT; + } + return 0; +} + +/** + * Wait until is possible to process some data + * @i2c: ocores I2C device instance + * + * Used when the device is in polling mode (interrupts disabled). + * + * Return: 0 on success, -ETIMEDOUT on timeout + */ +static int ocores_poll_wait(struct ocores_i2c *i2c) +{ + u8 mask; + int err; + + if (i2c->state == STATE_DONE || i2c->state == STATE_ERROR) { + /* transfer is over */ + mask = OCI2C_STAT_BUSY; + } else { + /* on going transfer */ + mask = OCI2C_STAT_TIP; + /* + * We wait for the data to be transferred (8bit), + * then we start polling on the ACK/NACK bit + */ + udelay((8 * 1000) / i2c->bus_clock_khz); + } + + /* + * once we are here we expect to get the expected result immediately + * so if after 1ms we timeout then something is broken. + */ + err = ocores_wait(i2c, OCI2C_STATUS, mask, 0, msecs_to_jiffies(1)); + if (err) + dev_warn(i2c->adap.dev.parent, + "%s: STATUS timeout, bit 0x%x did not clear in 1ms\n", + __func__, mask); + return err; +} + +/** + * It handles an IRQ-less transfer + * @i2c: ocores I2C device instance + * + * Even if IRQ are disabled, the I2C OpenCore IP behavior is exactly the same + * (only that IRQ are not produced). This means that we can re-use entirely + * ocores_isr(), we just add our polling code around it. + * + * It can run in atomic context + */ +static void ocores_process_polling(struct ocores_i2c *i2c) +{ + while (1) { + irqreturn_t ret; + int err; + + err = ocores_poll_wait(i2c); + if (err) { + i2c->state = STATE_ERROR; + break; /* timeout */ + } + + ret = ocores_isr(-1, i2c); + if (ret == IRQ_NONE) + break; /* all messages have been transferred */ + } +} + +static int ocores_xfer_core(struct ocores_i2c *i2c, + struct i2c_msg *msgs, int num, + bool polling) +{ + int ret; + u8 ctrl; + + ctrl = oc_getreg(i2c, OCI2C_CONTROL); + if (polling) + oc_setreg(i2c, OCI2C_CONTROL, ctrl & ~OCI2C_CTRL_IEN); + else + oc_setreg(i2c, OCI2C_CONTROL, ctrl | OCI2C_CTRL_IEN); + + i2c->msg = msgs; + i2c->pos = 0; + i2c->nmsgs = num; + i2c->state = STATE_START; + + oc_setreg(i2c, OCI2C_DATA, i2c_8bit_addr_from_msg(i2c->msg)); + oc_setreg(i2c, OCI2C_CMD, OCI2C_CMD_START); + + if (polling) { + ocores_process_polling(i2c); + } else { + ret = wait_event_timeout(i2c->wait, + (i2c->state == STATE_ERROR) || + (i2c->state == STATE_DONE), HZ); + if (ret == 0) { + ocores_process_timeout(i2c); + return -ETIMEDOUT; + } + } + + return (i2c->state == STATE_DONE) ? num : -EIO; +} + +static int ocores_xfer_polling(struct i2c_adapter *adap, + struct i2c_msg *msgs, int num) +{ + return ocores_xfer_core(i2c_get_adapdata(adap), msgs, num, true); +} + +static int ocores_xfer(struct i2c_adapter *adap, + struct i2c_msg *msgs, int num) +{ + return ocores_xfer_core(i2c_get_adapdata(adap), msgs, num, false); +} + +static int ocores_init(struct device *dev, struct ocores_i2c *i2c) +{ + int prescale; + int diff; + u8 ctrl = oc_getreg(i2c, OCI2C_CONTROL); + + /* make sure the device is disabled */ + ctrl &= ~(OCI2C_CTRL_EN | OCI2C_CTRL_IEN); + oc_setreg(i2c, OCI2C_CONTROL, ctrl); + + prescale = (i2c->ip_clock_khz / (5 * i2c->bus_clock_khz)) - 1; + prescale = clamp(prescale, 0, 0xffff); + + diff = i2c->ip_clock_khz / (5 * (prescale + 1)) - i2c->bus_clock_khz; + if (abs(diff) > i2c->bus_clock_khz / 10) { + dev_err(dev, + "Unsupported clock settings: core: %d KHz, bus: %d KHz\n", + i2c->ip_clock_khz, i2c->bus_clock_khz); + return -EINVAL; + } + + oc_setreg(i2c, OCI2C_PRELOW, prescale & 0xff); + oc_setreg(i2c, OCI2C_PREHIGH, prescale >> 8); + + /* Init the device */ + oc_setreg(i2c, OCI2C_CMD, OCI2C_CMD_IACK); + oc_setreg(i2c, OCI2C_CONTROL, ctrl | OCI2C_CTRL_EN); + + return 0; +} + + +static u32 ocores_func(struct i2c_adapter *adap) +{ + return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL; +} + +static struct i2c_algorithm ocores_algorithm = { + .master_xfer = ocores_xfer, + .functionality = ocores_func, +}; + +static const struct i2c_adapter ocores_adapter = { + .owner = THIS_MODULE, + .name = "i2c-ocores", + .class = I2C_CLASS_DEPRECATED, + .algo = &ocores_algorithm, + .nr = -1, +}; + +static const struct of_device_id ocores_i2c_match[] = { + { + .compatible = "opencores,i2c-ocores", + .data = (void *)TYPE_OCORES, + }, + { + .compatible = "aeroflexgaisler,i2cmst", + .data = (void *)TYPE_GRLIB, + }, + {}, +}; +MODULE_DEVICE_TABLE(of, ocores_i2c_match); + +#ifdef CONFIG_OF +/* + * Read and write functions for the GRLIB port of the controller. Registers are + * 32-bit big endian and the PRELOW and PREHIGH registers are merged into one + * register. The subsequent registers have their offsets decreased accordingly. + */ +static u8 oc_getreg_grlib(struct ocores_i2c *i2c, int reg) +{ + u32 rd; + int rreg = reg; + + if (reg != OCI2C_PRELOW) + rreg--; + rd = ioread32be(i2c->base + (rreg << i2c->reg_shift)); + if (reg == OCI2C_PREHIGH) + return (u8)(rd >> 8); + else + return (u8)rd; +} + +static void oc_setreg_grlib(struct ocores_i2c *i2c, int reg, u8 value) +{ + u32 curr, wr; + int rreg = reg; + + if (reg != OCI2C_PRELOW) + rreg--; + if (reg == OCI2C_PRELOW || reg == OCI2C_PREHIGH) { + curr = ioread32be(i2c->base + (rreg << i2c->reg_shift)); + if (reg == OCI2C_PRELOW) + wr = (curr & 0xff00) | value; + else + wr = (((u32)value) << 8) | (curr & 0xff); + } else { + wr = value; + } + iowrite32be(wr, i2c->base + (rreg << i2c->reg_shift)); +} + +static int ocores_i2c_of_probe(struct platform_device *pdev, + struct ocores_i2c *i2c) +{ + struct device_node *np = pdev->dev.of_node; + const struct of_device_id *match; + u32 val; + u32 clock_frequency; + bool clock_frequency_present; + + if (of_property_read_u32(np, "reg-shift", &i2c->reg_shift)) { + /* no 'reg-shift', check for deprecated 'regstep' */ + if (!of_property_read_u32(np, "regstep", &val)) { + if (!is_power_of_2(val)) { + dev_err(&pdev->dev, "invalid regstep %d\n", + val); + return -EINVAL; + } + i2c->reg_shift = ilog2(val); + dev_warn(&pdev->dev, + "regstep property deprecated, use reg-shift\n"); + } + } + + clock_frequency_present = !of_property_read_u32(np, "clock-frequency", + &clock_frequency); + i2c->bus_clock_khz = 100; + + i2c->clk = devm_clk_get(&pdev->dev, NULL); + + if (!IS_ERR(i2c->clk)) { + int ret = clk_prepare_enable(i2c->clk); + + if (ret) { + dev_err(&pdev->dev, + "clk_prepare_enable failed: %d\n", ret); + return ret; + } + i2c->ip_clock_khz = clk_get_rate(i2c->clk) / 1000; + if (clock_frequency_present) + i2c->bus_clock_khz = clock_frequency / 1000; + } + + if (i2c->ip_clock_khz == 0) { + if (of_property_read_u32(np, "opencores,ip-clock-frequency", + &val)) { + if (!clock_frequency_present) { + dev_err(&pdev->dev, + "Missing required parameter 'opencores,ip-clock-frequency'\n"); + clk_disable_unprepare(i2c->clk); + return -ENODEV; + } + i2c->ip_clock_khz = clock_frequency / 1000; + dev_warn(&pdev->dev, + "Deprecated usage of the 'clock-frequency' property, please update to 'opencores,ip-clock-frequency'\n"); + } else { + i2c->ip_clock_khz = val / 1000; + if (clock_frequency_present) + i2c->bus_clock_khz = clock_frequency / 1000; + } + } + + of_property_read_u32(pdev->dev.of_node, "reg-io-width", + &i2c->reg_io_width); + + match = of_match_node(ocores_i2c_match, pdev->dev.of_node); + if (match && (long)match->data == TYPE_GRLIB) { + dev_dbg(&pdev->dev, "GRLIB variant of i2c-ocores\n"); + i2c->setreg = oc_setreg_grlib; + i2c->getreg = oc_getreg_grlib; + } + + return 0; +} +#else +#define ocores_i2c_of_probe(pdev, i2c) -ENODEV +#endif + +static int ocores_i2c_probe(struct platform_device *pdev) +{ + struct ocores_i2c *i2c; + struct ocores_i2c_platform_data *pdata; + struct resource *res; + int irq; + int ret; + int i; + + i2c = devm_kzalloc(&pdev->dev, sizeof(*i2c), GFP_KERNEL); + if (!i2c) + return -ENOMEM; + + spin_lock_init(&i2c->process_lock); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (res) { + i2c->base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(i2c->base)) + return PTR_ERR(i2c->base); + } else { + res = platform_get_resource(pdev, IORESOURCE_IO, 0); + if (!res) + return -EINVAL; + i2c->iobase = res->start; + if (!devm_request_region(&pdev->dev, res->start, + resource_size(res), + pdev->name)) { + dev_err(&pdev->dev, "Can't get I/O resource.\n"); + return -EBUSY; + } + i2c->setreg = oc_setreg_io_8; + i2c->getreg = oc_getreg_io_8; + } + + pdata = dev_get_platdata(&pdev->dev); + if (pdata) { + i2c->adapter_nr = pdata->adapter_nr; + i2c->reg_shift = pdata->reg_shift; + i2c->reg_io_width = pdata->reg_io_width; + i2c->ip_clock_khz = pdata->clock_khz; + if (pdata->bus_khz) + i2c->bus_clock_khz = pdata->bus_khz; + else + i2c->bus_clock_khz = 100; + } else { + ret = ocores_i2c_of_probe(pdev, i2c); + if (ret) + return ret; + } + + if (i2c->reg_io_width == 0) + i2c->reg_io_width = 1; /* Set to default value */ + + if (!i2c->setreg || !i2c->getreg) { + bool be = pdata ? pdata->big_endian : + of_device_is_big_endian(pdev->dev.of_node); + + switch (i2c->reg_io_width) { + case 1: + i2c->setreg = oc_setreg_8; + i2c->getreg = oc_getreg_8; + break; + + case 2: + i2c->setreg = be ? oc_setreg_16be : oc_setreg_16; + i2c->getreg = be ? oc_getreg_16be : oc_getreg_16; + break; + + case 4: + i2c->setreg = be ? oc_setreg_32be : oc_setreg_32; + i2c->getreg = be ? oc_getreg_32be : oc_getreg_32; + break; + + default: + dev_err(&pdev->dev, "Unsupported I/O width (%d)\n", + i2c->reg_io_width); + ret = -EINVAL; + goto err_clk; + } + } + + init_waitqueue_head(&i2c->wait); + + irq = platform_get_irq(pdev, 0); + if (irq == -ENXIO) { + ocores_algorithm.master_xfer = ocores_xfer_polling; + } else { + if (irq < 0) + return irq; + } + + if (ocores_algorithm.master_xfer != ocores_xfer_polling) { + ret = devm_request_irq(&pdev->dev, irq, ocores_isr, 0, + pdev->name, i2c); + if (ret) { + dev_err(&pdev->dev, "Cannot claim IRQ\n"); + goto err_clk; + } + } + + ret = ocores_init(&pdev->dev, i2c); + if (ret) + goto err_clk; + + /* hook up driver to tree */ + platform_set_drvdata(pdev, i2c); + i2c->adap = ocores_adapter; + i2c_set_adapdata(&i2c->adap, i2c); + i2c->adap.dev.parent = &pdev->dev; + i2c->adap.dev.of_node = pdev->dev.of_node; + + if (pdata) { + i2c->adap.nr = i2c->adapter_nr; + } + + /* add i2c adapter to i2c tree */ + ret = i2c_add_numbered_adapter(&i2c->adap); + if (ret) + goto err_clk; + + /* add in known devices to the bus */ + if (pdata) { + for (i = 0; i < pdata->num_devices; i++) + i2c_new_device(&i2c->adap, pdata->devices + i); + } + + return 0; + +err_clk: + clk_disable_unprepare(i2c->clk); + return ret; +} + +static int ocores_i2c_remove(struct platform_device *pdev) +{ + struct ocores_i2c *i2c = platform_get_drvdata(pdev); + u8 ctrl = oc_getreg(i2c, OCI2C_CONTROL); + + /* disable i2c logic */ + ctrl &= ~(OCI2C_CTRL_EN | OCI2C_CTRL_IEN); + oc_setreg(i2c, OCI2C_CONTROL, ctrl); + + /* remove adapter & data */ + i2c_del_adapter(&i2c->adap); + + if (!IS_ERR(i2c->clk)) + clk_disable_unprepare(i2c->clk); + + return 0; +} + +#ifdef CONFIG_PM_SLEEP +static int ocores_i2c_suspend(struct device *dev) +{ + struct ocores_i2c *i2c = dev_get_drvdata(dev); + u8 ctrl = oc_getreg(i2c, OCI2C_CONTROL); + + /* make sure the device is disabled */ + ctrl &= ~(OCI2C_CTRL_EN | OCI2C_CTRL_IEN); + oc_setreg(i2c, OCI2C_CONTROL, ctrl); + + if (!IS_ERR(i2c->clk)) + clk_disable_unprepare(i2c->clk); + return 0; +} + +static int ocores_i2c_resume(struct device *dev) +{ + struct ocores_i2c *i2c = dev_get_drvdata(dev); + + if (!IS_ERR(i2c->clk)) { + unsigned long rate; + int ret = clk_prepare_enable(i2c->clk); + + if (ret) { + dev_err(dev, + "clk_prepare_enable failed: %d\n", ret); + return ret; + } + rate = clk_get_rate(i2c->clk) / 1000; + if (rate) + i2c->ip_clock_khz = rate; + } + return ocores_init(dev, i2c); +} + +static SIMPLE_DEV_PM_OPS(ocores_i2c_pm, ocores_i2c_suspend, ocores_i2c_resume); +#define OCORES_I2C_PM (&ocores_i2c_pm) +#else +#define OCORES_I2C_PM NULL +#endif + +static struct platform_driver ocores_i2c_driver = { + .probe = ocores_i2c_probe, + .remove = ocores_i2c_remove, + .driver = { + .name = "ocores-i2c-polling", + .of_match_table = ocores_i2c_match, + .pm = OCORES_I2C_PM, + }, +}; + +module_platform_driver(ocores_i2c_driver); + +MODULE_AUTHOR("Peter Korsgaard "); +MODULE_DESCRIPTION("OpenCores I2C bus driver"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:ocores-i2c"); \ No newline at end of file diff --git a/platform/broadcom/sonic-platform-modules-cel/questone2bd/modules/i2c-ocores-polling.h b/platform/broadcom/sonic-platform-modules-cel/questone2bd/modules/i2c-ocores-polling.h new file mode 100644 index 000000000000..d47d16406068 --- /dev/null +++ b/platform/broadcom/sonic-platform-modules-cel/questone2bd/modules/i2c-ocores-polling.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * i2c-ocores-polling.h - definitions for the i2c-ocores interface + * + * Peter Korsgaard + */ + +#ifndef _LINUX_I2C_OCORES_H +#define _LINUX_I2C_OCORES_H + +struct ocores_i2c_platform_data { + u32 reg_shift; /* register offset shift value */ + u32 reg_io_width; /* register io read/write width */ + u32 clock_khz; /* input clock in kHz */ + u32 bus_khz; /* bus clock in kHz */ + bool big_endian; /* registers are big endian */ + int adapter_nr; /* Desire adapter number, -1 to use dynamic number */ + u8 num_devices; /* number of devices in the devices list */ + struct i2c_board_info const *devices; /* devices connected to the bus */ +}; + +#endif /* _LINUX_I2C_OCORES_H */ \ No newline at end of file diff --git a/platform/broadcom/sonic-platform-modules-cel/questone2bd/modules/switchboard_fpga.c b/platform/broadcom/sonic-platform-modules-cel/questone2bd/modules/switchboard_fpga.c index 8dbf68f077fa..575f52251711 100644 --- a/platform/broadcom/sonic-platform-modules-cel/questone2bd/modules/switchboard_fpga.c +++ b/platform/broadcom/sonic-platform-modules-cel/questone2bd/modules/switchboard_fpga.c @@ -25,7 +25,7 @@ */ #ifndef TEST_MODE -#define MOD_VERSION "0.5.5" +#define MOD_VERSION "0.6.0" #else #define MOD_VERSION "TEST" #endif @@ -53,6 +53,7 @@ #include #include #include +#include "i2c-ocores-polling.h" static int majorNumber; @@ -362,6 +363,29 @@ static struct device* fpgafwdev = NULL; // < The device-driver device struct #define check(REG) #endif +/* I2C ocore configurations */ +#define OCORE_REGSHIFT 2 +#define OCORE_IP_CLK_khz 62500 +#define OCORE_BUS_CLK_khz 100 +#define OCORE_REG_IO_WIDTH 1 + +/* Resource IOMEM of i2c bus 6 */ +static struct resource fpga_i2c_6_res = { + .start = 0x8A0, + .end = 0x8BF, + .flags = IORESOURCE_MEM, +}; + +struct ocores_i2c_platform_data fpga_i2c_6_data = { + .reg_shift = OCORE_REGSHIFT, + .reg_io_width = OCORE_REG_IO_WIDTH, + .clock_khz = OCORE_IP_CLK_khz, + .bus_khz = OCORE_BUS_CLK_khz, + .big_endian = false, + .num_devices = 0, + .devices = NULL, +}; + static struct mutex fpga_i2c_master_locks[I2C_MASTER_CH_TOTAL]; /* Store lasted switch address and channel */ static uint16_t fpga_i2c_lasted_access_port[I2C_MASTER_CH_TOTAL]; @@ -414,7 +438,7 @@ static struct i2c_switch fpga_i2c_bus_dev[] = { {I2C_MASTER_CH_12, 0x70, 2, SFP, "SFP35"}, {I2C_MASTER_CH_12, 0x70, 3, SFP, "SFP36"}, {I2C_MASTER_CH_12, 0x70, 4, SFP, "SFP37"}, {I2C_MASTER_CH_12, 0x70, 5, SFP, "SFP38"}, {I2C_MASTER_CH_12, 0x70, 6, SFP, "SFP39"}, {I2C_MASTER_CH_12, 0x70, 7, SFP, "SFP40"}, - + /* BUS12 SFP Exported as virtual bus */ {I2C_MASTER_CH_12, 0x71, 0, SFP, "SFP41"}, {I2C_MASTER_CH_12, 0x71, 1, SFP, "SFP42"}, {I2C_MASTER_CH_12, 0x71, 2, SFP, "SFP43"}, {I2C_MASTER_CH_12, 0x71, 3, SFP, "SFP44"}, @@ -427,17 +451,17 @@ static struct i2c_switch fpga_i2c_bus_dev[] = { {I2C_MASTER_CH_12, 0x73, 2, QSFP, "QSFP7"}, {I2C_MASTER_CH_12, 0x73, 3, QSFP, "QSFP8"}, /* Vritual I2C adapters */ - {I2C_MASTER_CH_1, 0xFF, 0, NONE, "I2C_1"}, // FAN - {I2C_MASTER_CH_2, 0xFF, 0, NONE, "I2C_2"}, - {I2C_MASTER_CH_3, 0xFF, 0, NONE, "I2C_3"}, - {I2C_MASTER_CH_4, 0xFF, 0, NONE, "I2C_4"}, - {I2C_MASTER_CH_5, 0xFF, 0, NONE, "I2C_5"}, // BB - {I2C_MASTER_CH_6, 0xFF, 0, NONE, "I2C_6"}, - {I2C_MASTER_CH_7, 0xFF, 0, NONE, "I2C_7"}, // SW - - // NOTE: Two buses below are for front panel port debug - {I2C_MASTER_CH_11, 0xFF, 0, NONE, "I2C_11"}, // SFF - {I2C_MASTER_CH_12, 0xFF, 0, NONE, "I2C_12"}, + [56] = {I2C_MASTER_CH_1, 0xFF, 0, NONE, "I2C_1"}, // FAN + [57] = {I2C_MASTER_CH_2, 0xFF, 0, NONE, "I2C_2"}, + [58] = {I2C_MASTER_CH_3, 0xFF, 0, NONE, "I2C_3"}, + [59] = {I2C_MASTER_CH_4, 0xFF, 0, NONE, "I2C_4"}, + [60] = {I2C_MASTER_CH_5, 0xFF, 0, NONE, "I2C_5"}, // BB + [61] = {I2C_MASTER_CH_6, 0xFF, 0, NONE, "I2C_6"}, + [62] = {I2C_MASTER_CH_7, 0xFF, 0, NONE, "I2C_7"}, // SW + + // NOTE: Two buses below are for front panel port debug + [63] = {I2C_MASTER_CH_11, 0xFF, 0, NONE, "I2C_11"}, // SFF + [64] = {I2C_MASTER_CH_12, 0xFF, 0, NONE, "I2C_12"}, }; @@ -462,6 +486,7 @@ static struct fpga_device fpga_dev = { /* * struct questone2bd_fpga_data - Private data for questone2bd switchboard driver. + * @res_base: Base address of FPGA PCI region. * @sff_device: List of optical module device node. * @i2c_client: List of optical module I2C client device. * @i2c_adapter: List of I2C adapter populates by this driver. @@ -470,11 +495,13 @@ static struct fpga_device fpga_dev = { * @fpga_read_addr: Buffer point to FPGA's register. * @cpld1_read_addr: Buffer point to CPLD1's register. * @cpld2_read_addr: Buffer point to CPLD2's register. + * @volt_i2c_adap: platform device pointer of FPGA_I2C_MASTER_6. * * For *_read_addr, its temporary hold the register address to be read by * getreg sysfs, see *getreg() for more details. */ struct questone2bd_fpga_data { + resource_size_t res_base; struct device *sff_devices[SFF_PORT_TOTAL]; struct i2c_client *sff_i2c_clients[SFF_PORT_TOTAL]; struct i2c_adapter *i2c_adapter[VIRTUAL_I2C_PORT_LENGTH]; @@ -483,6 +510,7 @@ struct questone2bd_fpga_data { void __iomem * fpga_read_addr; uint8_t cpld1_read_addr; uint8_t cpld2_read_addr; + struct platform_device *volt_i2c_adap; }; struct sff_device_data { @@ -1176,9 +1204,9 @@ static ssize_t port_led_color_show(struct device *dev, struct device_attribute * if (err < 0) return err; return sprintf(buf, "%s %s\n", - led_color1 == 0x07 ? "off" : led_color1 == 0x06 ? "green" : led_color1 == 0x05 ? "amber" : led_color1 == 0x04 ? + led_color1 == 0x07 ? "off" : led_color1 == 0x06 ? "green" : led_color1 == 0x05 ? "amber" : led_color1 == 0x04 ? "both" : "alt-blink", - led_color1 == 0x07 ? "off" : led_color1 == 0x06 ? "green" : led_color1 == 0x05 ? "amber" : led_color1 == 0x04 ? + led_color1 == 0x07 ? "off" : led_color1 == 0x06 ? "green" : led_color1 == 0x05 ? "amber" : led_color1 == 0x04 ? "both" : "alt-blink"); } @@ -1292,7 +1320,7 @@ static void i2c_core_deinit(unsigned int master_bus,void __iomem *pci_bar){ * Return: 0 if success, error code less than zero if fails. */ static int i2c_xcvr_access(u8 register_address, unsigned int portid, u8 *data, char rw){ - + u16 dev_addr = 0; int err; unsigned int sw_cpld_lock_index = 0; @@ -1393,7 +1421,7 @@ static int i2c_wait_stop(struct i2c_adapter *a, unsigned long timeout, int writi break; } - if ( (Status & ( 1 << I2C_STAT_BUSY )) == 0 ) + if ( (Status & ( 1 << I2C_STAT_BUSY )) == 0 ) { dev_dbg(&a->dev," IF:%2.2X\n", Status); break; @@ -1459,7 +1487,7 @@ static int i2c_wait_ack(struct i2c_adapter *a, unsigned long timeout, int writin Status = ioread8(pci_bar + REG_STAT); dev_dbg(&a->dev,"ST:%2.2X\n", Status); - if ( (Status & ( 1 << I2C_STAT_TIP )) == 0 ) + if ( (Status & ( 1 << I2C_STAT_TIP )) == 0 ) { dev_dbg(&a->dev," IF:%2.2X\n", Status); break; @@ -1484,9 +1512,9 @@ static int i2c_wait_ack(struct i2c_adapter *a, unsigned long timeout, int writin return error; } - /** There is only one master in each bus. If this error happen something is + /** There is only one master in each bus. If this error happen something is * not normal in i2c transfer refer to: - * https://www.i2c-bus.org/i2c-primer/analysing-obscure-problems/master-reports-arbitration-lost + * https://www.i2c-bus.org/i2c-primer/analysing-obscure-problems/master-reports-arbitration-lost */ // Arbitration lost if (Status & (1 << I2C_STAT_AL)) { @@ -1755,7 +1783,7 @@ static int smbus_access(struct i2c_adapter *adapter, u16 addr, iowrite8(1 << I2C_CMD_RD, pci_bar + REG_CMD); } - + // Wait {A} error = i2c_wait_ack(adapter, 30, 0); if(nack_retry[master_bus - 1] == 1) @@ -1827,7 +1855,7 @@ static int fpga_i2c_access(struct i2c_adapter *adapter, u16 addr, if (dev_data == NULL) return -ESHUTDOWN; - + master_bus = dev_data->pca9548.master_bus; switch_addr = dev_data->pca9548.switch_addr; channel = dev_data->pca9548.channel; @@ -1970,7 +1998,7 @@ static int fpga_i2c_access(struct i2c_adapter *adapter, u16 addr, udelay(1); iowrite8( ioread8(fpga_dev.data_base_addr + 0x0108) | 0x70, fpga_dev.data_base_addr + 0x0108); } - // clear the last access port + // clear the last access port fpga_i2c_lasted_access_port[master_bus - 1] = 0; }else{ dev_crit(&adapter->dev, "I2C bus unrecoverable.\n"); @@ -1978,7 +2006,7 @@ static int fpga_i2c_access(struct i2c_adapter *adapter, u16 addr, } -release_unlock: +release_unlock: mutex_unlock(&fpga_i2c_master_locks[master_bus - 1]); dev_dbg(&adapter->dev,"switch ch %d of 0x%x -> ch %d of 0x%x\n", prev_ch, prev_switch, channel, switch_addr); return retval; @@ -2012,7 +2040,9 @@ static const struct i2c_algorithm questone2bd_i2c_algorithm = { * When bus_number_offset is -1, created adapter with dynamic bus number. * Otherwise create adapter at i2c bus = bus_number_offset + portid. */ -static struct i2c_adapter * questone2bd_i2c_init(struct platform_device *pdev, int portid, int bus_number_offset) +static struct i2c_adapter *i2c_adapter_init(struct platform_device *pdev, + int portid, + int bus_number_offset) { int error; @@ -2061,7 +2091,93 @@ static struct i2c_adapter * questone2bd_i2c_init(struct platform_device *pdev, i } return new_adapter; -}; +} + +/* + * Register an i2c-ocores platform device with its resources + * pdev platform device to get the private data. + * nr The desire bus number to use when create I2C adapter. + * Returns 0 if success, negative errno if error. + */ +static int ocores_i2c_init(struct platform_device *pdev, int nr) { + + int error = 0; + struct questone2bd_fpga_data *priv; + struct platform_device *device; + + priv = platform_get_drvdata(pdev); + fpga_i2c_6_res.start += priv->res_base; + fpga_i2c_6_res.end += priv->res_base; + fpga_i2c_6_data.adapter_nr = nr; + + device = platform_device_register_resndata(&pdev->dev, + "ocores-i2c-polling", + -1, + &fpga_i2c_6_res, + 1, + &fpga_i2c_6_data, + sizeof(fpga_i2c_6_data)); + if (IS_ERR(device)) + error = PTR_ERR(device); + + priv->volt_i2c_adap = device; + return error; +} + +static struct i2c_adapter *questone2bd_i2c_init(struct platform_device *pdev, + int portid, + int bus_number_offset) { + + /* Create a platform device of i2c opencores for master bus 6. + * Force return adapter as NULL. + */ + unsigned char master_bus; + int adapter_nr; + + adapter_nr = portid + bus_number_offset; + master_bus = fpga_i2c_bus_dev[portid].master_bus; + + if (master_bus == I2C_MASTER_CH_6) { + if (ocores_i2c_init(pdev, adapter_nr) < 0) { + dev_err(&pdev->dev, "Failed to register I2C_MASTER_CH_6 device\n"); + } + return NULL; + } + + return i2c_adapter_init(pdev, portid, bus_number_offset); +} + +/* + * Initialize upstream I2c switch of this adapter. By close all of it channel. + * @param adapter - an I2C adapter. + * @param prev_reset_switch - current success reset switch. + * + * Reset the PCA9548 switch that is not reset by *prev_reset_switch* before. + * If the current upstream switch match with *prev_reset_switch*, skip operation. + * Else write a command to PCA9548 and update *prev_reset_switch* variable. + */ +static void init_muxed_adapter(struct i2c_adapter *adapter, uint16_t *prev_reset_switch) { + + struct i2c_dev_data *dev_data; + unsigned char master_bus; + unsigned char switch_addr; + uint16_t current_switch; + + dev_data = i2c_get_adapdata(adapter); + master_bus = dev_data->pca9548.master_bus; + switch_addr = dev_data->pca9548.switch_addr; + + current_switch = ((master_bus << 8) | switch_addr); + + if (switch_addr != 0xFF) { + + if (*prev_reset_switch != current_switch) { + // Found the bus with PCA9548, trying to clear all switch in it. + smbus_access(adapter, switch_addr, 0x00, I2C_SMBUS_WRITE, 0x00, I2C_SMBUS_BYTE, NULL); + *prev_reset_switch = current_switch; + } + } +} // I/O resource need. static struct resource questone2bd_resources[] = { @@ -2115,6 +2231,9 @@ static int questone2bd_drv_probe(struct platform_device *pdev) if (!fpga_data) return -ENOMEM; + platform_set_drvdata(pdev, fpga_data); + fpga_data->res_base = fpga_dev.data_mmio_start; + // Set default read address to VERSION fpga_data->fpga_read_addr = fpga_dev.data_base_addr + FPGA_VERSION; fpga_data->cpld1_read_addr = 0x00; @@ -2233,7 +2352,7 @@ static int questone2bd_drv_probe(struct platform_device *pdev) for (portid_count = I2C_MASTER_CH_1; portid_count <= I2C_MASTER_CH_TOTAL; portid_count++){ if(!allow_unsafe_i2c_access){ - if( portid_count < I2C_MASTER_CH_7 || + if( portid_count < I2C_MASTER_CH_7 || portid_count == I2C_MASTER_CH_9 || portid_count == I2C_MASTER_CH_10 ) continue; } @@ -2305,48 +2424,45 @@ static int questone2bd_drv_probe(struct platform_device *pdev) } } - struct i2c_dev_data *dev_data; - unsigned char master_bus; - unsigned char switch_addr; - - dev_data = i2c_get_adapdata(fpga_data->i2c_adapter[portid_count]); - master_bus = dev_data->pca9548.master_bus; - switch_addr = dev_data->pca9548.switch_addr; - - if (switch_addr != 0xFF) { - - if (prev_i2c_switch != ( (master_bus << 8) | switch_addr) ) { - // Found the bus with PCA9548, trying to clear all switch in it. - smbus_access(fpga_data->i2c_adapter[portid_count], switch_addr, 0x00, I2C_SMBUS_WRITE, 0x00, I2C_SMBUS_BYTE, NULL); - prev_i2c_switch = ( master_bus << 8 ) | switch_addr; - } + if (fpga_data->i2c_adapter[portid_count]) { + init_muxed_adapter(fpga_data->i2c_adapter[portid_count], + &prev_i2c_switch); } + } return 0; } + static int questone2bd_drv_remove(struct platform_device *pdev) { int portid_count; struct sff_device_data *rem_data; struct i2c_dev_data *adap_data; + struct questone2bd_fpga_data *priv; + + priv = platform_get_drvdata(pdev); for (portid_count = 0; portid_count < SFF_PORT_TOTAL; portid_count++) { - sysfs_remove_link(&fpga_data->sff_devices[portid_count]->kobj, "i2c"); - i2c_unregister_device(fpga_data->sff_i2c_clients[portid_count]); + sysfs_remove_link(&priv->sff_devices[portid_count]->kobj, "i2c"); + i2c_unregister_device(priv->sff_i2c_clients[portid_count]); } for (portid_count = 0 ; portid_count < VIRTUAL_I2C_PORT_LENGTH ; portid_count++) { - if (fpga_data->i2c_adapter[portid_count] != NULL) { - info(KERN_INFO "<%x>", fpga_data->i2c_adapter[portid_count]); - adap_data = i2c_get_adapdata(fpga_data->i2c_adapter[portid_count]); - i2c_del_adapter(fpga_data->i2c_adapter[portid_count]); + if (!IS_ERR_OR_NULL(priv->i2c_adapter[portid_count])) { + info(KERN_INFO "<%x>", priv->i2c_adapter[portid_count]); + adap_data = i2c_get_adapdata(priv->i2c_adapter[portid_count]); + i2c_del_adapter(priv->i2c_adapter[portid_count]); } } + if (priv->volt_i2c_adap) { + platform_device_unregister(priv->volt_i2c_adap); + } + for (portid_count = I2C_MASTER_CH_1; portid_count <= I2C_MASTER_CH_TOTAL; portid_count++){ if(!allow_unsafe_i2c_access){ - if( portid_count < I2C_MASTER_CH_7 || + if( portid_count < I2C_MASTER_CH_7 || portid_count == I2C_MASTER_CH_9 || portid_count == I2C_MASTER_CH_10 ) continue; } @@ -2354,10 +2470,10 @@ static int questone2bd_drv_remove(struct platform_device *pdev) } for (portid_count = 0; portid_count < SFF_PORT_TOTAL; portid_count++) { - if (fpga_data->sff_devices[portid_count] != NULL) { - rem_data = dev_get_drvdata(fpga_data->sff_devices[portid_count]); - device_unregister(fpga_data->sff_devices[portid_count]); - put_device(fpga_data->sff_devices[portid_count]); + if (!IS_ERR_OR_NULL(priv->sff_devices[portid_count])) { + rem_data = dev_get_drvdata(priv->sff_devices[portid_count]); + device_unregister(priv->sff_devices[portid_count]); + put_device(priv->sff_devices[portid_count]); kfree(rem_data); } } @@ -2370,7 +2486,7 @@ static int questone2bd_drv_remove(struct platform_device *pdev) kobject_put(cpld1); kobject_put(cpld2); device_destroy(fpgafwclass, MKDEV(0, 0)); - devm_kfree(&pdev->dev, fpga_data); + devm_kfree(&pdev->dev, priv); return 0; } @@ -2408,11 +2524,6 @@ static int fpga_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) return err; } - if ((err = pci_request_regions(pdev, FPGA_PCI_NAME)) < 0) { - dev_err(dev, "pci_request_regions error %d\n", err); - goto pci_disable; - } - /* bar0: data mmio region */ fpga_dev.data_mmio_start = pci_resource_start(pdev, FPGA_PCI_BAR_NUM); fpga_dev.data_mmio_len = pci_resource_len(pdev, FPGA_PCI_BAR_NUM); @@ -2420,7 +2531,7 @@ static int fpga_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (!fpga_dev.data_base_addr) { dev_err(dev, "cannot iomap region of size %lu\n", (unsigned long)fpga_dev.data_mmio_len); - goto pci_release; + goto pci_disable; } dev_info(dev, "data_mmio iomap base = 0x%lx \n", (unsigned long)fpga_dev.data_base_addr); @@ -2437,14 +2548,14 @@ static int fpga_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) fpga_version = ioread32(fpga_dev.data_base_addr); printk(KERN_INFO "FPGA Version : %8.8x\n", fpga_version); if ((err = fpgafw_init()) < 0){ - goto pci_release; + goto pci_unmap; } platform_device_register(&questone2bd_dev); platform_driver_register(&questone2bd_drv); return 0; -pci_release: - pci_release_regions(pdev); +pci_unmap: + pci_iounmap(pdev, fpga_dev.data_base_addr); pci_disable: pci_disable_device(pdev); return err; @@ -2456,7 +2567,6 @@ static void fpga_pci_remove(struct pci_dev *pdev) platform_device_unregister(&questone2bd_dev); fpgafw_exit(); pci_iounmap(pdev, fpga_dev.data_base_addr); - pci_release_regions(pdev); pci_disable_device(pdev); printk(KERN_INFO "FPGA PCIe driver remove OK.\n"); };