shim.h

/**
 * Copyright (C) 2017-2018 Xilinx, Inc
 * Author: Sonal Santan
 * AWS HAL Driver layered on top of kernel drivers
 *
 * Code copied from SDAccel XDMA based HAL driver
 *
 * Licensed under the Apache License, Version 2.0 (the "License"). You may
 * not use this file except in compliance with the License. A copy of the
 * License is located at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations
 * under the License.
 */
#ifndef _XDMA_SHIM_H_
#define _XDMA_SHIM_H_

#include "xclhal.h"
#include "xclperf.h"
#include "drm.h"
#include <fstream>
#include <list>
#include <map>
#include <vector>
#include <string>
#include <mutex>
#include <cassert>

#ifndef INTERNAL_TESTING
#include "fpga_pci.h"
#include "fpga_mgmt.h"
#endif

// Work around GCC 4.8 + XDMA BAR implementation bugs
// With -O3 PCIe BAR read/write are not reliable hence force -O2 as max
// optimization level for pcieBarRead() and pcieBarWrite()
#if defined(__GNUC__) && defined(NDEBUG)
#define SHIM_O2 __attribute__ ((optimize("-O2")))
#else
#define SHIM_O2
#endif

namespace awsbwhal {


struct AddresRange;

std::ostream& operator<< (std::ostream &strm, const AddresRange &rng);

/**
 * Simple tuple struct to store non overlapping address ranges: address and size
 */
struct AddresRange : public std::pair<uint64_t, size_t> {
  // size will be zero when we are looking up an address that was passed by the user
  AddresRange(uint64_t addr, size_t size = 0) : std::pair<uint64_t, size_t>(std::make_pair(addr, size)) {
    //std::cout << "CTOR(" << addr << ',' << size << ")\n";
  }
  AddresRange(AddresRange && rhs) : std::pair<uint64_t, size_t>(std::move(rhs)) {
    //std::cout << "MOVE CTOR(" << rhs.first << ',' << rhs.second << ")\n";
  }

  AddresRange(const AddresRange &rhs) = delete;
  AddresRange& operator=(const AddresRange &rhs) = delete;

  // Comparison operator is useful when using AddressRange as a key in std::map
  // Note one operand in the comparator may have only the address without the size
  // However both operands in the comparator will not have zero size
  bool operator < (const AddresRange& other) const {
    //std::cout << *this << " < " << other << "\n";
    if ((this->second != 0) && (other.second != 0))
      // regular ranges
      return (this->first < other.first);
    if (other.second == 0)
      // second range just has an address
      // (1000, 100) < (1200, 0)
      // (1000, 100) < (1100, 0) first range ends at 1099
      return ((this->first + this->second) <= other.first);
    assert(this->second == 0);
    // this range just has an address
    // (1100, 0) < (1200, 100)
    return (this->first < other.first);
  }
};

/**
 * Simple map of address range to its bo handle and mapped virtual address
 */
static const std::pair<unsigned, char *> mNullValue = std::make_pair(0xffffffff, nullptr);
class RangeTable {
  std::map<AddresRange, std::pair<unsigned, char *>> mTable;
  mutable std::mutex mMutex;
public:
  void insert(uint64_t addr, size_t size, std::pair<unsigned, char *> bo) {
    // assert(find(addr) == 0xffffffff);
    std::lock_guard<std::mutex> lock(mMutex);
    mTable[AddresRange(addr, size)] = bo;
  }

  std::pair<unsigned, char *> erase(uint64_t addr) {
    std::lock_guard<std::mutex> lock(mMutex);
    std::map<AddresRange, std::pair<unsigned, char *>>::const_iterator i = mTable.find(AddresRange(addr));
    if (i == mTable.end())
      return mNullValue;
    std::pair<unsigned, char *> result = i->second;
    mTable.erase(i);
    return result;
  }

  std::pair<unsigned, char *> find(uint64_t addr) const {
    std::lock_guard<std::mutex> lock(mMutex);
    std::map<AddresRange, std::pair<unsigned, char *>>::const_iterator i = mTable.find(AddresRange(addr));
    if (i == mTable.end())
      return mNullValue;
    return i->second;
  }
};


    // Memory alignment for DDR and AXI-MM trace access
    template <typename T> class AlignedAllocator {
        void *mBuffer;
        size_t mCount;
    public:
        T *getBuffer() {
            return (T *)mBuffer;
        }

        size_t size() const {
            return mCount * sizeof(T);
        }

        AlignedAllocator(size_t alignment, size_t count) : mBuffer(0), mCount(count) {
            if (posix_memalign(&mBuffer, alignment, count * sizeof(T))) {
                mBuffer = 0;
            }
        }
        ~AlignedAllocator() {
            if (mBuffer)
                free(mBuffer);
        }
    };

    const uint64_t mNullAddr = 0xffffffffffffffffull;
    const uint64_t mNullBO = 0xffffffff;

    // XDMA Shim
    class AwsXcl{

        struct ELARecord {
            unsigned mStartAddress;
            unsigned mEndAddress;
            unsigned mDataCount;

            std::streampos mDataPos;
            ELARecord() : mStartAddress(0), mEndAddress(0),
                          mDataCount(0), mDataPos(0) {}
        };

        typedef std::list<ELARecord> ELARecordList;

        typedef std::list<std::pair<uint64_t, uint64_t> > PairList;

    public:
        //Sarab: Added for HAL2 XOCL Driver support
        //int xclGetErrorStatus(xclErrorStatus *info); Not supported for AWS
        bool xclUnlockDevice();
        unsigned int xclAllocBO(size_t size, xclBOKind domain, unsigned flags);
        unsigned int xclAllocUserPtrBO(void *userptr, size_t size, unsigned flags);
        void xclFreeBO(unsigned int boHandle);
        int xclWriteBO(unsigned int boHandle,
            const void *src, size_t size, size_t seek);
        int xclReadBO(unsigned int boHandle,
            void *dst, size_t size, size_t skip);
        void *xclMapBO(unsigned int boHandle, bool write);
        int xclSyncBO(unsigned int boHandle, xclBOSyncDirection dir,
            size_t size, size_t offset);
        int xclExportBO(unsigned int boHandle);
        unsigned int xclImportBO(int fd, unsigned flags);
        int xclGetBOProperties(unsigned int boHandle, xclBOProperties *properties);
        ssize_t xclUnmgdPread(unsigned flags, void *buf,
            size_t count, uint64_t offset);
        ssize_t xclUnmgdPwrite(unsigned flags, const void *buf,
            size_t count, uint64_t offset);


        // Bitstreams
        int xclGetXclBinIdFromSysfs(uint64_t &xclbinid);
        int xclLoadXclBin(const xclBin *buffer);
        int xclLoadAxlf(const axlf *buffer);
        int xclUpgradeFirmware(const char *fileName);
        int xclUpgradeFirmware2(const char *file1, const char* file2);
        //int xclUpgradeFirmwareXSpi(const char *fileName, int device_index=0); Not supported by AWS
        int xclTestXSpi(int device_index);
        int xclBootFPGA();
        int xclRemoveAndScanFPGA();
        int resetDevice(xclResetKind kind);
        int xclReClock2(unsigned short region, const unsigned short *targetFreqMHz);

        // Raw read/write
        size_t xclWrite(xclAddressSpace space, uint64_t offset, const void *hostBuf, size_t size);
        size_t xclRead(xclAddressSpace space, uint64_t offset, void *hostBuf, size_t size);

        // Buffer management
        uint64_t xclAllocDeviceBuffer(size_t size);
        uint64_t xclAllocDeviceBuffer2(size_t size, xclMemoryDomains domain, unsigned flags);
        void xclFreeDeviceBuffer(uint64_t buf);
        size_t xclCopyBufferHost2Device(uint64_t dest, const void *src, size_t size, size_t seek);
        size_t xclCopyBufferDevice2Host(void *dest, uint64_t src, size_t size, size_t skip);

        // Performance monitoring
        // Control
        double xclGetDeviceClockFreqMHz();
        double xclGetReadMaxBandwidthMBps();
        double xclGetWriteMaxBandwidthMBps();
        //void xclSetOclRegionProfilingNumberSlots(uint32_t numSlots);
        void xclSetProfilingNumberSlots(xclPerfMonType type, uint32_t numSlots);
        size_t xclPerfMonClockTraining(xclPerfMonType type);
        // Counters
        size_t xclPerfMonStartCounters(xclPerfMonType type);
        size_t xclPerfMonStopCounters(xclPerfMonType type);
        size_t xclPerfMonReadCounters(xclPerfMonType type, xclCounterResults& counterResults);
        //debug related
        uint32_t getCheckerNumberSlots(int type);
        uint32_t getIPCountAddrNames(int type, uint64_t *baseAddress, std::string * portNames);
        size_t xclDebugReadCounters(xclDebugCountersResults* debugResult);
        size_t xclDebugReadCheckers(xclDebugCheckersResults* checkerResult);
        void readDebugIpLayout();

        // Trace
        size_t xclPerfMonStartTrace(xclPerfMonType type, uint32_t startTrigger);
        size_t xclPerfMonStopTrace(xclPerfMonType type);
        uint32_t xclPerfMonGetTraceCount(xclPerfMonType type);
        size_t xclPerfMonReadTrace(xclPerfMonType type, xclTraceResultsVector& traceVector);

        // Sanity checks
        int xclGetDeviceInfo2(xclDeviceInfo2 *info);
        static AwsXcl *handleCheck(void *handle);
        static unsigned xclProbe();
        bool xclLockDevice();
        unsigned getTAG() const {
            return mTag;
        }
        bool isGood() const;

        ~AwsXcl();
        AwsXcl(unsigned index, const char *logfileName, xclVerbosityLevel verbosity);

    private:

        size_t xclReadModifyWrite(uint64_t offset, const void *hostBuf, size_t size);
        size_t xclReadSkipCopy(uint64_t offset, void *hostBuf, size_t size);
        bool zeroOutDDR();

        bool isXPR() const {
          return ((mDeviceInfo.mSubsystemId >> 12) == 4);
        }

        bool isMultipleOCLClockSupported() {
          unsigned dsaNum = ((mDeviceInfo.mDeviceId << 16) | mDeviceInfo.mSubsystemId);
          // 0x82384431 : TUL KU115 4ddr 3.1 DSA
          return ((dsaNum == 0x82384431)  || (dsaNum == 0x82384432))? true : false;
        }

        bool isUltraScale() const {
            return (mDeviceInfo.mDeviceId & 0x8000);
        }

        // Core DMA code
        SHIM_O2 int pcieBarRead(int bar_num, unsigned long long offset, void* buffer, unsigned long long length);
        SHIM_O2 int pcieBarWrite(int bar_num, unsigned long long offset, const void* buffer, unsigned long long length);
        int freezeAXIGate();
        int freeAXIGate();

        // PROM flashing
        int prepare(unsigned startAddress, unsigned endAddress);
        int program(std::ifstream& mcsStream, const ELARecord& record);
        int program(std::ifstream& mcsStream);
        int waitForReady(unsigned code, bool verbose = true);
        int waitAndFinish(unsigned code, unsigned data, bool verbose = true);

        //XSpi flashing.
        bool prepareXSpi();
        int programXSpi(std::ifstream& mcsStream, const ELARecord& record);
        int programXSpi(std::ifstream& mcsStream);
        bool waitTxEmpty();
        bool isFlashReady();
        //bool windDownWrites();
        bool bulkErase();
        bool sectorErase(unsigned Addr);
        bool writeEnable();
#if 0
	bool dataTransfer(bool read);
#endif
        bool readPage(unsigned addr, uint8_t readCmd = 0xff);
        bool writePage(unsigned addr, uint8_t writeCmd = 0xff);
        unsigned readReg(unsigned offset);
        int writeReg(unsigned regOffset, unsigned value);
        bool finalTransfer(uint8_t *sendBufPtr, uint8_t *recvBufPtr, int byteCount);
        bool getFlashId();
        //All remaining read /write register commands can be issued through this function.
        bool readRegister(unsigned commandCode, unsigned bytes);
        bool writeRegister(unsigned commandCode, unsigned value, unsigned bytes);
        bool select4ByteAddressMode();
        bool deSelect4ByteAddressMode();


        // Performance monitoring helper functions
        bool isDSAVersion(unsigned majorVersion, unsigned minorVersion, bool onlyThisVersion);
        unsigned getBankCount();
        uint64_t getHostTraceTimeNsec();
        uint64_t getPerfMonBaseAddress(xclPerfMonType type, uint32_t slotNum);
        uint64_t getPerfMonFifoBaseAddress(xclPerfMonType type, uint32_t fifonum);
        uint64_t getPerfMonFifoReadBaseAddress(xclPerfMonType type, uint32_t fifonum);
        uint32_t getPerfMonNumberSlots(xclPerfMonType type);
        uint32_t getPerfMonNumberSamples(xclPerfMonType type);
        uint32_t getPerfMonNumberFifos(xclPerfMonType type);
        uint32_t getPerfMonByteScaleFactor(xclPerfMonType type);
        uint8_t  getPerfMonShowIDS(xclPerfMonType type);
        uint8_t  getPerfMonShowLEN(xclPerfMonType type);
        uint32_t getPerfMonSlotStartBit(xclPerfMonType type, uint32_t slotnum);
        uint32_t getPerfMonSlotDataWidth(xclPerfMonType type, uint32_t slotnum);
        size_t resetFifos(xclPerfMonType type);
        uint32_t bin2dec(std::string str, int start, int number);
        uint32_t bin2dec(const char * str, int start, int number);
        std::string dec2bin(uint32_t n);
        std::string dec2bin(uint32_t n, unsigned bits);
        static std::string getDSAName(unsigned short deviceId, unsigned short subsystemId);

    private:
        // This is a hidden signature of this class and helps in preventing
        // user errors when incorrect pointers are passed in as handles.
        const unsigned mTag;
        const int mBoardNumber;
        const size_t maxDMASize;
        bool mLocked;
        const uint64_t mOffsets[XCL_ADDR_SPACE_MAX];
        int mUserHandle;
#ifdef INTERNAL_TESTING
        int mMgtHandle;
#else
        pci_bar_handle_t ocl_kernel_bar;     // AppPF BAR0 for OpenCL kernels
        pci_bar_handle_t sda_mgmt_bar;       // MgmtPF BAR4, for SDAccel Perf mon etc
        pci_bar_handle_t ocl_global_mem_bar; // AppPF BAR4
#endif
        uint32_t mMemoryProfilingNumberSlots;
        uint32_t mOclRegionProfilingNumberSlots;
        std::string mDevUserName;

        // Information extracted from platform linker
        bool mIsDebugIpLayoutRead = false;
        bool mIsDeviceProfiling = false;
        uint64_t mPerfMonFifoCtrlBaseAddress;
        uint64_t mPerfMonFifoReadBaseAddress;
        uint64_t mPerfMonBaseAddress[XSPM_MAX_NUMBER_SLOTS];
        std::string mPerfMonSlotName[XSPM_MAX_NUMBER_SLOTS];

        char *mUserMap;
        std::ofstream mLogStream;
        xclVerbosityLevel mVerbosity;
        std::string mBinfile;
        ELARecordList mRecordList;
        xclDeviceInfo2 mDeviceInfo;
        RangeTable mLegacyAddressTable;

#ifndef INTERNAL_TESTING
        int sleepUntilLoaded( std::string afi );
        int checkAndSkipReload( char *afi_id, fpga_mgmt_image_info *info );
        int loadDefaultAfiIfCleared( void );
#endif
    public:
        static const unsigned TAG;
    };
}

#endif