Skip to content

Commit

Permalink
[#166]Fetch updates one page (default 10 items) at a time.
Browse files Browse the repository at this point in the history
  • Loading branch information
@stellanl
stellanl committed Apr 15, 2015
1 parent 519c3e0 commit c661017
Show file tree
Hide file tree
Showing 9 changed files with 167 additions and 706 deletions.
4 changes: 2 additions & 2 deletions android/AkvoRSR/AndroidManifest.xml
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="org.akvo.rsr.up"
android:versionCode="22"
android:versionName="1.3.9"
android:versionCode="23"
android:versionName="1.4.0Beta"
>

<uses-sdk
Expand Down
21 changes: 12 additions & 9 deletions android/AkvoRSR/src/org/akvo/rsr/up/service/GetProjectDataService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ protected void onHandleIntent(Intent intent) {
RsrDbAdapter ad = new RsrDbAdapter(this);
Downloader dl = new Downloader();
String errMsg = null;
boolean noimages = SettingsUtil.ReadBoolean(this, "setting_delay_image_fetch", false);
boolean fetchImages = !SettingsUtil.ReadBoolean(this, "setting_delay_image_fetch", false);
String host = SettingsUtil.host(this);

ad.open();
Expand All @@ -80,8 +80,8 @@ protected void onHandleIntent(Intent intent) {
dl.fetchProject(this,
ad,
new URL(SettingsUtil.host(this) +
String.format(ConstantUtil.FETCH_PROJ_URL_PATTERN,id)));
broadcastProgress(0, i++, projects);
String.format(ConstantUtil.FETCH_PROJ_URL_PATTERN, id)));
broadcastProgress(0, ++i, projects);
}
if (mFetchCountries) {
// TODO: rarely changes, so only fetch countries if we never did that
Expand All @@ -93,21 +93,24 @@ protected void onHandleIntent(Intent intent) {
if (mFetchUpdates) {
int j = 0;
for (String projId : user.getPublishedProjIds()) {
dl.fetchUpdateListRestApi(this, //TODO: use _extra for fewer fetches, as country and user data is included
dl.fetchUpdateListRestApiPaged(this, //TODO: use _extra for fewer fetches, as country and user data is included
new URL(host + String.format(ConstantUtil.FETCH_UPDATE_URL_PATTERN, projId))
);
broadcastProgress(1, j++, projects);
broadcastProgress(1, ++j, projects);
}
}

} catch (FileNotFoundException e) {
Log.e(TAG, "Cannot find:", e);
errMsg = getResources().getString(R.string.errmsg_not_found_on_server) + e.getMessage();
} catch (Exception e) {
} catch (Exception e) {//get e==null here!!!!
Log.e(TAG, "Bad updates fetch:", e);
errMsg = getResources().getString(R.string.errmsg_update_fetch_failed) + e.getMessage();
if (e != null) {
errMsg = getResources().getString(R.string.errmsg_update_fetch_failed) + e.getMessage();
} else {
errMsg = getResources().getString(R.string.errmsg_update_fetch_failed) + "NULL exception";
}
}

if (mFetchUsers) { //Remove this once we use the _extra update API
// Fetch missing user data for authors of the updates.
// This API requires authorization
Expand Down Expand Up @@ -165,7 +168,7 @@ protected void onHandleIntent(Intent intent) {

broadcastProgress(1, 100, 100);

if (!noimages) {
if (fetchImages) {
try {
dl.fetchNewThumbnails(this,
host,
Expand Down
2 changes: 1 addition & 1 deletion android/AkvoRSR/src/org/akvo/rsr/up/service/SignInService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ protected void onHandleIntent(Intent intent) {
}
catch (Exception e) {
i2.putExtra(ConstantUtil.SERVICE_ERRMSG_KEY, getResources().getString(R.string.errmsg_signin_failed) + e.getMessage());
Log.e(TAG,"SignIn() error:",e);
Log.e(TAG,"SignInService() error:", e);
}

//broadcast completion
Expand Down
5 changes: 3 additions & 2 deletions android/AkvoRSR/src/org/akvo/rsr/up/util/ConstantUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,10 @@ public class ConstantUtil {
public static final String AUTH_URL = "/auth/token/";
public static final String API_KEY_PATTERN = "&api_key=%s&username=%s";
public static final String POST_UPDATE_URL = "/rest/v1/project_update/?format=xml";
public static final String FETCH_UPDATE_URL_PATTERN = "/rest/v1/project_update/?format=xml&limit=1000&project=%s";
// public static final String FETCH_UPDATE_URL_PATTERN = "/rest/v1/project_update/?format=xml&limit=1000&project=%s";
public static final String FETCH_UPDATE_URL_PATTERN = "/rest/v1/project_update/?format=xml&project=%s";//use default limit
public static final String VERIFY_UPDATE_PATTERN = "/rest/v1/project_update/?format=xml&uuid=%s&limit=2";
public static final String FETCH_PROJ_URL_PATTERN = "/rest/v1/project_up/%s/?format=xml&image_thumb_name=up&width=100"; //ask for thumbnail size
public static final String FETCH_PROJ_URL_PATTERN = "/rest/v1/project_up/%s/?format=xml&image_thumb_name=up&image_thumb_up_width=100"; //ask for thumbnail size
public static final String FETCH_COUNTRIES_URL = "/api/v1/country/?format=xml&limit=0";
public static final String FETCH_PROJ_COUNT_URL = "/api/v1/project/?format=xml&limit=0&partnerships__organisation=%s";
public static final String PROJECT_PATH_PATTERN = "/api/v1/project/%s/";
Expand Down
126 changes: 86 additions & 40 deletions android/AkvoRSR/src/org/akvo/rsr/up/util/Downloader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,11 +40,9 @@
import org.akvo.rsr.up.xml.CountryListHandler;
import org.akvo.rsr.up.xml.OrganisationHandler;
import org.akvo.rsr.up.xml.ProjectExtraRestHandler;
import org.akvo.rsr.up.xml.ProjectListHandler;
import org.akvo.rsr.up.xml.UpdateExtraRestListHandler;
import org.akvo.rsr.up.xml.UpdateRestHandler;
import org.akvo.rsr.up.xml.UpdateRestListHandler;
import org.akvo.rsr.up.xml.UpdateListHandler;
import org.akvo.rsr.up.xml.UserListHandler;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
Expand Down Expand Up @@ -96,13 +94,27 @@ public static class UnresolvedPostException extends IOException {
private static final long serialVersionUID = -630304430323100535L;
}

public static class FailedPostException extends Exception {
public FailedPostException(String string) {
super(string);
}
public static class FailedPostException extends Exception {
public FailedPostException(String string) {
super(string);
}

private static final long serialVersionUID = -8091570663513780467L;
}
private static final long serialVersionUID = -8091570663513780467L;
}


public static class FailedFetchException extends Exception {

/**
*
*/
private static final long serialVersionUID = 2355800973621221158L;

public FailedFetchException(String string) {
super(string);
}

}


/**
Expand Down Expand Up @@ -151,37 +163,6 @@ public void fetchProject(Context ctx, RsrDbAdapter dba, URL url) throws ParserCo
}


/**
* populates the updates table in the db from a server URL
* Typically the url will specify updates for a single project.
*
* @param ctx
* @param url
* @throws ParserConfigurationException
* @throws SAXException
* @throws IOException
*/
public void fetchUpdateList(Context ctx, URL url) throws ParserConfigurationException, SAXException, IOException {

/* Get a SAXParser from the SAXPArserFactory. */
SAXParserFactory spf = SAXParserFactory.newInstance();
SAXParser sp = spf.newSAXParser();

/* Get the XMLReader of the SAXParser we created. */
XMLReader xr = sp.getXMLReader();
/* Create a new ContentHandler and apply it to the XML-Reader*/
UpdateListHandler myUpdateListHandler = new UpdateListHandler(new RsrDbAdapter(ctx), true, false);
xr.setContentHandler(myUpdateListHandler);
/* Parse the xml-data from our URL. */
xr.parse(new InputSource(url.openStream()));
/* Parsing has finished. */

/* Check if anything went wrong. */
err = myUpdateListHandler.getError();
Log.i(TAG, "Fetched " + myUpdateListHandler.getCount() + " updates");
}


/**
* populates the updates table in the db from a server URL
* in the REST API
Expand Down Expand Up @@ -211,6 +192,7 @@ public Date fetchUpdateListRestApi(Context ctx, URL url) throws ParserConfigurat
XMLReader xr = sp.getXMLReader();
/* Create a new ContentHandler and apply it to the XML-Reader*/
UpdateRestListHandler myUpdateListHandler = new UpdateRestListHandler(new RsrDbAdapter(ctx), true);
//the following will need to be called in a loop to get it page by page, or it would probably take too long for server
// UpdateExtraRestListHandler myUpdateListHandler = new UpdateExtraRestListHandler(new RsrDbAdapter(ctx), true, serverVersion);
xr.setContentHandler(myUpdateListHandler);
/* Parse the xml-data from our URL. */
Expand All @@ -230,6 +212,70 @@ public Date fetchUpdateListRestApi(Context ctx, URL url) throws ParserConfigurat
}


/**
* populates the updates table in the db from a server URL
* in the REST API
* Typically the url will specify updates for a single project.
* should eventually call project_update_extra call
* this will avoid having to call country/org/user APIs separately
*
* @param ctx
* @param url
* @throws ParserConfigurationException
* @throws SAXException
* @throws IOException
* @throws FailedFetchException
*/
public Date fetchUpdateListRestApiPaged(Context ctx, URL url) throws ParserConfigurationException, SAXException, IOException, FailedFetchException {
Date serverDate = null;
User user = SettingsUtil.getAuthUser(ctx);
int total = 0;
//the is called in a loop to get it page by page, otherwise it would take too long for server
//and it would not scale beyond 1000 updates in any case
while (url != null) {
HttpRequest h = HttpRequest.get(url).connectTimeout(10000); //10 sec timeout
h.header("Authorization", "Token " + user.getApiKey()); //This API needs authorization
int code = h.code();//evaluation starts the exchange
if (code == 200) {
String serverVersion = h.header(ConstantUtil.SERVER_VERSION_HEADER);
serverDate = new Date(h.date());
SAXParserFactory spf = SAXParserFactory.newInstance();
XMLReader xr = spf.newSAXParser().getXMLReader();
UpdateRestListHandler xmlHandler = new UpdateRestListHandler(new RsrDbAdapter(ctx), true);
// UpdateExtraRestListHandler myUpdateListHandler = new UpdateExtraRestListHandler(new RsrDbAdapter(ctx), true, serverVersion);
xr.setContentHandler(xmlHandler);
/* Parse the xml-data from our URL. */
xr.parse(new InputSource(h.stream()));
/* Parsing has finished. */
/* Check if anything went wrong. */
err = xmlHandler.getError();
Log.i(TAG, "Fetched " + xmlHandler.getCount() + " updates; target total = "+ xmlHandler.getTotalCount());
total += xmlHandler.getCount();
if (xmlHandler.getNextUrl().length() == 0) { //string needs to be trimmed from whitespace
url = null;//we are done
} else {
try {
url = new URL(xmlHandler.getNextUrl());//TODO is this xml-escaped?
}
catch (MalformedURLException e) {
url = null;
}
}

} else if (code == 404) {
url = null;//we are done
} else {
//Vanilla case is 403 forbidden on an auth failure
Log.e(TAG, "Fetch update list HTTP error code:" + code);
Log.e(TAG, h.body());
throw new FailedFetchException("Unexpected server response " + code);
}
}
Log.i(TAG, "Grand total of " + total + " updates");
return serverDate;
}


/**
* Verify status at server of a single Update
*
Expand Down Expand Up @@ -655,7 +701,7 @@ public static boolean postXmlUpdateStreaming(String urlTemplate, Update update,
//Just long+lat for location. We do not currently do reverse geocoding in the app.
//Country used to be mandatory, but that was changed
final String locationTemplate = "<locations><list-item><longitude>%s</longitude><latitude>%s</latitude></list-item></locations>";
final boolean simulateUnresolvedPost = true;
final boolean simulateUnresolvedPost = false;

boolean allSent = false;
try {
Expand Down
Loading

0 comments on commit c661017

Please sign in to comment.