[OSM-dev] Server Timeouts - JOSM and the forthcoming big uploads via the API

[Dave Hansen]

On Mon, 2007-08-13 at 09:36 +0100, Andy Robinson wrote:
> While uploading TIGER data last week it became apparent that when the API is
> very quiet (like I am just about the only person uploading data) then using
> JOSM the server times out as regularly as clockwork at around 250 node
> uploads. At busier times the upload might be slower but you may get past the
> 250 point because someone else suffers the timeout on the specific call
> rather than you.

I'm running into the same things with the TIGER uploads.  The following
patch at least retries failed operations, like 500 errors and connection
timeouts.  It needs to back off a bit more, but I think the concept is
workable.

I would love not having to depend on JOSM to do the work, though.  I'm
in the process of restarting two counties that got screwed up during and
upload.  I accidentally pulled JOSM's .jar file out from underneath it
while it was running, and it got really cranky.

Index: src/org/openstreetmap/josm/io/OsmServerWriter.java
===================================================================
--- src/org/openstreetmap/josm/io/OsmServerWriter.java	(revision 302)
+++ src/org/openstreetmap/josm/io/OsmServerWriter.java	(working copy)
@@ -12,6 +12,7 @@
 import java.net.HttpURLConnection;
 import java.net.URL;
 import java.net.UnknownHostException;
+import java.net.SocketTimeoutException;
 import java.util.Collection;
 import java.util.LinkedList;
 
@@ -66,12 +67,14 @@
 		NameVisitor v = new NameVisitor();
 		try {
 			for (OsmPrimitive osm : list) {
+				int progress = Main.pleaseWaitDlg.progress.getValue();
 				if (cancel)
 					return;
 				osm.visit(v);
-				Main.pleaseWaitDlg.currentAction.setText(tr("Upload {0} {1} ({2})...", tr(v.className), v.name, osm.id));
+				Main.pleaseWaitDlg.currentAction.setText(tr("Upload {0} {1} (id: {2}) {3}% {4}/{5}...",
+						tr(v.className), v.name, osm.id, 100.0*progress/list.size(), progress, list.size()));
 				osm.visit(this);
-				Main.pleaseWaitDlg.progress.setValue(Main.pleaseWaitDlg.progress.getValue()+1);
+				Main.pleaseWaitDlg.progress.setValue(progress+1);
 			}
 		} catch (RuntimeException e) {
 			e.printStackTrace();
@@ -150,8 +153,8 @@
 	 * @param addBody <code>true</code>, if the whole primitive body should be added.
 	 * 		<code>false</code>, if only the id is encoded.
 	 */
-	private void sendRequest(String requestMethod, String urlSuffix,
-			OsmPrimitive osm, boolean addBody) {
+	private void __sendRequest(String requestMethod, String urlSuffix,
+			OsmPrimitive osm, boolean addBody, int retries) {
 		try {
 			String version = Main.pref.get("osm-server.version", "0.4");
 			URL url = new URL(
@@ -165,7 +168,17 @@
 			activeConnection.setRequestMethod(requestMethod);
 			if (addBody)
 				activeConnection.setDoOutput(true);
-			activeConnection.connect();
+
+			boolean timed_out = true;
+			while (timed_out) {
+				try {
+					activeConnection.connect();
+					timed_out = false;
+				} catch(SocketTimeoutException e) {
+					if (retries-- <= 0)
+						throw e;
+				}
+			}
 
 			if (addBody) {
 				OutputStream out = activeConnection.getOutputStream();
@@ -182,6 +195,11 @@
 			if (retCode == 410 && requestMethod.equals("DELETE"))
 				return; // everything fine.. was already deleted.
 			if (retCode != 200) {
+				if (retries >= 0) {
+					retries--;
+					System.out.println("retrying ("+retries+" left)");
+					__sendRequest(requestMethod, urlSuffix, osm, addBody, retries);
+				}
 				// Look for a detailed error message from the server
 				if (activeConnection.getHeaderField("Error") != null)
 					retMsg += "\n" + activeConnection.getHeaderField("Error");
@@ -199,7 +217,11 @@
 				return; // assume cancel
 			if (e instanceof RuntimeException)
 				throw (RuntimeException)e;
-			throw new RuntimeException(e.getMessage(), e);
+			throw new RuntimeException(e.getMessage()+ " " + e.getClass().getCanonicalName(), e);
 		}
 	}
+	private void sendRequest(String requestMethod, String urlSuffix,
+			OsmPrimitive osm, boolean addBody) {
+		__sendRequest(requestMethod, urlSuffix, osm, addBody, 5);
+	}
 }


-- Dave