CSV import enhancements, 1.6 release, etc.

CHANGES

@@ -1,3 +1,21 @@
+2003-03-30  #################### PortaBase 1.6 ####################
+
+2003-03-30  JMB  CSV import enhancements, menu update bugfix
+
+    If an imported row doesn't have enough columns and the last few
+    columns are strings or notes, PortaBase now treats those fields as
+    empty strings.  (There often aren't enough delimiters at the end of
+    lines ending with blank fields when exporting from programs like Excel,
+    this just corrects for that.)
+
+    CSV files can now be imported as either UTF-8 or Latin-1; this avoids
+    the need for users importing files exported from other programs in
+    Latin-1 to use another program to perform an encoding conversion.
+
+    In version 1.5, making changes in the format editor would clear the
+    view, sorting, and filter lists from the menus without restoring them;
+    they are now populated correctly again.
+
 2003-03-27  JMB  Japanese UI translation, better CSV import error messages
 
     The entire PortaBase interface has now been translated into Japanese.

README.txt

@@ -20,7 +20,8 @@ The main features PortaBase currently has are:
 - View summary statistics for columns (total, average, min, max, etc.)
 - Import data from CSV, XML, and MobileDB files
 - Export data to CSV and XML files
-- Command-line format conversions (to and from XML, from MobileDB)
+- Command-line format conversions (to and from XML, to and from CSV,
+  from MobileDB)
 - Data file encryption
 - Unicode support
 - Pick any available font to use throughout the application

commandline.cpp

@@ -69,6 +69,14 @@ int CommandLine::fromOtherFormat(int argc, char **argv)
         }
         numArgs += 2;
     }
+    int encIndex = args.findIndex("-e");
+    if (encIndex != -1) {
+        if (argc < encIndex + 4) {
+            printUsage();
+            return 1;
+        }
+        numArgs += 2;
+    }
     if (argc != numArgs) {
         printUsage();
         return 1;
@@ -116,7 +124,11 @@ int CommandLine::fromOtherFormat(int argc, char **argv)
         error = utils.importXML(sourceFile, db);
     }
     else if (fromcsv) {
-        QStringList result = db->importFromCSV(sourceFile);
+        QString encoding = "UTF-8";
+        if (encIndex != -1) {
+            encoding = args[encIndex + 1];
+        }
+        QStringList result = db->importFromCSV(sourceFile, encoding);
         int count = result.count();
         if (count > 0) {
             error = result[0];
@@ -266,6 +278,8 @@ void CommandLine::printUsage()
     printf("    -v viewname (apply this view before exporting)\n");
     printf("    -s sortname (apply this sorting before exporting)\n");
     printf("    -f filtername (apply this filter before exporting)\n");
+    printf("  There is one option for fromcsv:\n");
+    printf("    -e encoding (UTF-8 or Latin-1; default is UTF-8)\n");
     printf("  When using fromcsv, \"tofile\" must be an existing PortaBase file.\n");
     printf("  Specify -h or --help to receive this message\n");
 }

csvutils.cpp

@@ -12,11 +12,9 @@
 #include <qfile.h>
 #include <qobject.h>
 #include <qregexp.h>
-#include <qstringlist.h>
 #include <qtextstream.h>
 #include "csvutils.h"
 #include "database.h"
-#include "datatypes.h"
 
 CSVUtils::CSVUtils() : m_textquote('"'), m_delimiter(',')
 {
@@ -28,27 +26,34 @@ CSVUtils::~CSVUtils()
 
 }
 
-QStringList CSVUtils::parseFile(QString filename, Database *db)
+QStringList CSVUtils::parseFile(const QString &filename,
+                                const QString &encoding, Database *db)
 {
+    initializeCounts(db);
     QFile f(filename);
     QStringList returnVal;
     if (!f.open(IO_ReadOnly)) {
         returnVal.append(QObject::tr("Unable to open file"));
         return returnVal;
     }
     QTextStream input(&f);
-    input.setEncoding(QTextStream::UnicodeUTF8);
-    int rowNum = 1;
-    QString message = "";
+    if (encoding == "Latin-1") {
+        input.setEncoding(QTextStream::Latin1);
+    }
+    else {
+        input.setEncoding(QTextStream::UnicodeUTF8);
+    }
+    rowNum = 1;
+    message = "";
     enum { S_START, S_QUOTED_FIELD, S_MAYBE_END_OF_QUOTED_FIELD,
            S_END_OF_QUOTED_FIELD, S_MAYBE_NORMAL_FIELD,
            S_NORMAL_FIELD } state = S_START;
     QChar x;
-    QString rowString = "";
-    QStringList row;
+    rowString = "";
+    row.clear();
     QString field = "";
     // store IDs of added rows; if there's an error, delete them
-    IntList addedIds;
+    addedIds.clear();
     while (!input.atEnd()) {
         input >> x; // read one char
 
@@ -75,17 +80,9 @@ QStringList CSVUtils::parseFile(QString filename, Database *db)
                     // row ended on a delimiter (empty cell)
                     row.append("");
                     field = "";
-                    int rowId = 0;
-                    message = db->addRow(row, &rowId);
-                    if (message != "") {
-                        message = QObject::tr("Error in row %1").arg(rowNum)
-                                  + "\n" + message;
+                    if (!addRow(db)) {
                         break;
                     }
-                    addedIds.append(rowId);
-                    row.clear();
-                    rowString = "";
-                    rowNum++;
                 }
             }
             else {
@@ -110,17 +107,9 @@ QStringList CSVUtils::parseFile(QString filename, Database *db)
                 row.append(field);
                 field = "";
                 if (x == '\n') {
-                    int rowId = 0;
-                    message = db->addRow(row, &rowId);
-                    if (message != "") {
-                        message = QObject::tr("Error in row %1").arg(rowNum)
-                                  + "\n" + message;
+                    if (!addRow(db)) {
                         break;
                     }
-                    addedIds.append(rowId);
-                    row.clear();
-                    rowString = "";
-                    rowNum++;
                 }
                 state = S_START;
             }
@@ -133,17 +122,9 @@ QStringList CSVUtils::parseFile(QString filename, Database *db)
                 row.append(field);
                 field = "";
                 if (x == '\n') {
-                    int rowId = 0;
-                    message = db->addRow(row, &rowId);
-                    if (message != "") {
-                        message = QObject::tr("Error in row %1").arg(rowNum)
-                                  + "\n" + message;
+                    if (!addRow(db)) {
                         break;
                     }
-                    addedIds.append(rowId);
-                    row.clear();
-                    rowString = "";
-                    rowNum++;
                 }
                 state = S_START;
             }
@@ -166,17 +147,9 @@ QStringList CSVUtils::parseFile(QString filename, Database *db)
             else if (x == '\n') {
                 row.append(field);
                 field = "";
-                int rowId = 0;
-                message = db->addRow(row, &rowId);
-                if (message != "") {
-                    message = QObject::tr("Error in row %1").arg(rowNum)
-                              + "\n" + message;
+                if (!addRow(db)) {
                     break;
                 }
-                addedIds.append(rowId);
-                row.clear();
-                rowString = "";
-                rowNum++;
             }
             else {
                 field += x;
@@ -189,12 +162,7 @@ QStringList CSVUtils::parseFile(QString filename, Database *db)
     if (message == "" && row.count() > 0) {
         // last line doesn't end with '\n'
         row.append(field);
-        int rowId = 0;
-        message = db->addRow(row, &rowId);
-        if (message != "") {
-            message = QObject::tr("Error in row %1").arg(rowNum)
-            + "\n" + message;
-        }
+        addRow(db);
     }
     if (message != "") {
         // an error was encountered; delete any rows that were added
@@ -232,3 +200,41 @@ QString CSVUtils::encodeCell(QString content)
     result += content.replace(QRegExp("\""), "\"\"");
     return result + "\"";
 }
+
+void CSVUtils::initializeCounts(Database *db)
+{
+    QStringList colNames = db->listColumns();
+    colCount = colNames.count();
+    endStringCount = 0;
+    int i;
+    for (i = colCount - 1; i > -1; i--) {
+        int type = db->getType(colNames[i]);
+        if (type != STRING && type != NOTE) {
+            break;
+        }
+        endStringCount++;
+    }
+}
+
+bool CSVUtils::addRow(Database *db)
+{
+    int countDiff = colCount - row.count();
+    if (countDiff > 0 && countDiff <= endStringCount) {
+        // last columns may have been blank in Excel or whatever...
+        int i;
+        for (i = 0; i < countDiff; i++) {
+            row.append("");
+        }
+    }
+    int rowId = 0;
+    message = db->addRow(row, &rowId);
+    if (message != "") {
+        message = QObject::tr("Error in row %1").arg(rowNum) + "\n" + message;
+        return FALSE;
+    }
+    addedIds.append(rowId);
+    row.clear();
+    rowString = "";
+    rowNum++;
+    return TRUE;
+}

csvutils.h

@@ -12,23 +12,37 @@
 #ifndef CSVUTILS_H
 #define CSVUTILS_H
 
+#include <qstringlist.h>
+#include "datatypes.h"
+
 class Database;
 class QString;
-class QStringList;
 
 class CSVUtils
 {
 public:
     CSVUtils();
     ~CSVUtils();
 
-    QStringList parseFile(QString filename, Database *db);
+    QStringList parseFile(const QString &filename, const QString &encoding,
+                          Database *db);
     QString encodeRow(QStringList row);
     QString encodeCell(QString content);
 
+private:
+    void initializeCounts(Database *db);
+    bool addRow(Database *db);
+
 private:
     QChar m_textquote;
     QChar m_delimiter;
+    int colCount;
+    int endStringCount;
+    int rowNum;
+    QString message;
+    QString rowString;
+    QStringList row;
+    IntList addedIds;
 };
 
 #endif

database.cpp

@@ -1423,10 +1423,11 @@ QPixmap Database::getCheckBoxPixmap(int checked)
     }
 }
 
-QStringList Database::importFromCSV(QString filename)
+QStringList Database::importFromCSV(const QString &filename,
+                                    const QString &encoding)
 {
     CSVUtils csv;
-    return csv.parseFile(filename, this);
+    return csv.parseFile(filename, encoding, this);
 }
 
 void Database::exportToXML(QString filename, c4_View &fullView,

database.h

@@ -128,7 +128,8 @@ class Database
     QString timeToString(int time);
     QString parseTimeString(QString value, bool *ok);
     void setShowSeconds(bool show);
-    QStringList importFromCSV(QString filename);
+    QStringList importFromCSV(const QString &filename,
+                              const QString &encoding);
     void exportToXML(QString filename, c4_View &fullView,
                      c4_View &filteredView, QStringList cols);
     void setGlobalInfo(const QString &view, const QString &sorting,

dbeditor.cpp

@@ -17,6 +17,7 @@
 #include <qlistview.h>
 #include <qmessagebox.h>
 #include <qpushbutton.h>
+#include <qregexp.h>
 #include <qvbox.h>
 #include "columneditor.h"
 #include "database.h"
@@ -311,6 +312,9 @@ void DBEditor::updateTable()
         int type = ceType (temp[i]);
         QString typeString = getTypeString(type);
         QString defaultVal = QString::fromUtf8(ceDefault (temp[i]));
+        if (type == NOTE) {
+            defaultVal = defaultVal.replace(QRegExp("\n"), " ");
+        }
         if (i == 0) {
             last = new QListViewItem(table, name, typeString, defaultVal);
         }

debian/changelog

@@ -1,3 +1,9 @@
+portabase (1.6-1) unstable; urgency=low
+
+  * New upstream version
+
+ -- Jeremy Bowman <jmbowman@alum.mit.edu>  Sun, 30 Mar 2003 15:56:53 -0500
+
 portabase (1.5-1) unstable; urgency=low
 
   * Initial Release.

debian/portabase.1

@@ -99,6 +99,12 @@ Apply the named sorting before exporting.
 .TP
 \fB\-f\fP \fI<filter>\fP
 Apply the named filter before exporting.
+.PP
+The fromcsv command supports one option:
+.TP
+\fB\-e\fP \fI<encoding>\fP
+Specifies the text encoding of the imported CSV file; options are UTF-8
+(the default) and Latin-1.
 .SH OPTIONS
 \fIPortaBase\fP accepts the following options:
 .TP

desktop/importdialog.cpp

@@ -11,6 +11,7 @@
 
 #include <qfiledialog.h>
 #include <qfileinfo.h>
+#include <qinputdialog.h>
 #include <qmessagebox.h>
 #include <qobject.h>
 #include <qstringlist.h>
@@ -54,10 +55,23 @@ bool ImportDialog::exec()
         QFileInfo info(file);
         QPEApplication::setDocumentDir(info.dirPath(TRUE));
     }
+
+    QStringList encodings;
+    encodings.append("UTF-8");
+    encodings.append("Latin-1");
+    bool ok;
+    QString encoding = QInputDialog::getItem(QObject::tr("Import"),
+                                            QObject::tr("Text encoding") + ":",
+                                            encodings, 0, FALSE, &ok,
+                                            parentWidget);
+    if (!ok) {
+        return FALSE;
+    }
+
     QString error;
     QString data = "";
     if (source == CSV_FILE) {
-        QStringList result = db->importFromCSV(file);
+        QStringList result = db->importFromCSV(file, encoding);
         int count = result.count();
         if (count > 0) {
             error = result[0];