001: /*******************************************************************************
002: * Licensed to the Apache Software Foundation (ASF) under one
003: * or more contributor license agreements. See the NOTICE file
004: * distributed with this work for additional information
005: * regarding copyright ownership. The ASF licenses this file
006: * to you under the Apache License, Version 2.0 (the
007: * "License"); you may not use this file except in compliance
008: * with the License. You may obtain a copy of the License at
009: *
010: * http://www.apache.org/licenses/LICENSE-2.0
011: *
012: * Unless required by applicable law or agreed to in writing,
013: * software distributed under the License is distributed on an
014: * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
015: * KIND, either express or implied. See the License for the
016: * specific language governing permissions and limitations
017: * under the License.
018: *******************************************************************************/package org.ofbiz.webtools;
019:
020: import java.text.NumberFormat;
021: import java.util.HashMap;
022: import java.util.Iterator;
023: import java.util.LinkedList;
024: import java.util.List;
025: import java.util.ArrayList;
026: import java.util.Collection;
027: import java.util.TreeSet;
028: import java.util.Locale;
029: import java.util.Map;
030: import java.io.File;
031: import java.io.InputStream;
032: import java.io.StringReader;
033: import java.io.StringWriter;
034: import java.io.FileReader;
035: import java.io.PrintWriter;
036: import java.io.BufferedWriter;
037: import java.io.OutputStreamWriter;
038: import java.io.FileOutputStream;
039: import java.io.IOException;
040: import java.io.FileNotFoundException;
041: import java.net.URL;
042: import java.net.MalformedURLException;
043:
044: import javolution.util.FastList;
045:
046: import org.ofbiz.base.util.Debug;
047: import org.ofbiz.base.util.StringUtil;
048: import org.ofbiz.base.util.UtilMisc;
049: import org.ofbiz.base.util.UtilProperties;
050: import org.ofbiz.base.util.UtilURL;
051: import org.ofbiz.base.util.UtilValidate;
052: import org.ofbiz.base.util.UtilDateTime;
053: import org.ofbiz.entity.GenericDelegator;
054: import org.ofbiz.entity.GenericEntityException;
055: import org.ofbiz.entity.GenericValue;
056: import org.ofbiz.entity.util.EntityDataLoader;
057: import org.ofbiz.entity.util.EntityListIterator;
058: import org.ofbiz.entity.util.EntitySaxReader;
059: import org.ofbiz.entity.model.ModelReader;
060: import org.ofbiz.entity.model.ModelEntity;
061: import org.ofbiz.entity.model.ModelViewEntity;
062: import org.ofbiz.security.Security;
063: import org.ofbiz.service.DispatchContext;
064: import org.ofbiz.service.LocalDispatcher;
065: import org.ofbiz.service.ServiceUtil;
066:
067: import org.xml.sax.InputSource;
068: import freemarker.template.*;
069: import freemarker.ext.dom.NodeModel;
070: import freemarker.ext.beans.BeansWrapper;
071:
072: /**
073: * WebTools Services
074: */
075:
076: public class WebToolsServices {
077:
078: public static final String module = WebToolsServices.class
079: .getName();
080:
081: public static Map entityImport(DispatchContext dctx, Map context) {
082: GenericValue userLogin = (GenericValue) context
083: .get("userLogin");
084: Security security = dctx.getSecurity();
085: if (!security.hasPermission("ENTITY_MAINT", userLogin)) {
086: return ServiceUtil.returnError(UtilProperties.getMessage(
087: "WebtoolsUiLabels", "WebtoolsPermissionError",
088: (Locale) context.get("locale")));
089: }
090:
091: LocalDispatcher dispatcher = dctx.getDispatcher();
092:
093: List messages = new ArrayList();
094:
095: String filename = (String) context.get("filename");
096: String fmfilename = (String) context.get("fmfilename");
097: String fulltext = (String) context.get("fulltext");
098: boolean isUrl = (String) context.get("isUrl") != null;
099: String mostlyInserts = (String) context.get("mostlyInserts");
100: String maintainTimeStamps = (String) context
101: .get("maintainTimeStamps");
102: String createDummyFks = (String) context.get("createDummyFks");
103:
104: Integer txTimeout = (Integer) context.get("txTimeout");
105:
106: if (txTimeout == null) {
107: txTimeout = new Integer(7200);
108: }
109: InputSource ins = null;
110: URL url = null;
111:
112: // #############################
113: // The filename to parse is prepared
114: // #############################
115: if (filename != null && filename.length() > 0) {
116: try {
117: url = isUrl ? new URL(filename) : UtilURL
118: .fromFilename(filename);
119: InputStream is = url.openStream();
120: ins = new InputSource(is);
121: } catch (MalformedURLException mue) {
122: return ServiceUtil
123: .returnError("ERROR: invalid file name ("
124: + filename + "): " + mue.getMessage());
125: } catch (IOException ioe) {
126: return ServiceUtil
127: .returnError("ERROR reading file name ("
128: + filename + "): " + ioe.getMessage());
129: } catch (Exception exc) {
130: return ServiceUtil
131: .returnError("ERROR: reading file name ("
132: + filename + "): " + exc.getMessage());
133: }
134: }
135:
136: // #############################
137: // The text to parse is prepared
138: // #############################
139: if (fulltext != null && fulltext.length() > 0) {
140: StringReader sr = new StringReader(fulltext);
141: ins = new InputSource(sr);
142: }
143:
144: // #############################
145: // FM Template
146: // #############################
147: String s = null;
148: if (UtilValidate.isNotEmpty(fmfilename) && ins != null) {
149: FileReader templateReader = null;
150: try {
151: templateReader = new FileReader(fmfilename);
152: } catch (FileNotFoundException e) {
153: return ServiceUtil
154: .returnError("ERROR reading template file ("
155: + fmfilename + "): " + e.getMessage());
156: }
157:
158: StringWriter outWriter = new StringWriter();
159:
160: Template template = null;
161: try {
162: Configuration conf = org.ofbiz.base.util.template.FreeMarkerWorker
163: .getDefaultOfbizConfig();
164: template = new Template("FMImportFilter",
165: templateReader, conf);
166: Map fmcontext = new HashMap();
167:
168: NodeModel nodeModel = NodeModel.parse(ins);
169: fmcontext.put("doc", nodeModel);
170: BeansWrapper wrapper = BeansWrapper
171: .getDefaultInstance();
172: TemplateHashModel staticModels = wrapper
173: .getStaticModels();
174: fmcontext.put("Static", staticModels);
175:
176: template.process(fmcontext, outWriter);
177: s = outWriter.toString();
178: } catch (Exception ex) {
179: return ServiceUtil
180: .returnError("ERROR processing template file ("
181: + fmfilename + "): " + ex.getMessage());
182: }
183: }
184:
185: // #############################
186: // The parsing takes place
187: // #############################
188: if (s != null || fulltext != null || url != null) {
189: try {
190: Map inputMap = UtilMisc.toMap("mostlyInserts",
191: mostlyInserts, "createDummyFks",
192: createDummyFks, "maintainTimeStamps",
193: maintainTimeStamps, "txTimeout", txTimeout,
194: "userLogin", userLogin);
195: if (s != null) {
196: inputMap.put("xmltext", s);
197: } else {
198: if (fulltext != null) {
199: inputMap.put("xmltext", fulltext);
200: } else {
201: inputMap.put("url", url);
202: }
203: }
204: Map outputMap = dispatcher.runSync(
205: "parseEntityXmlFile", inputMap);
206: if (ServiceUtil.isError(outputMap)) {
207: return ServiceUtil.returnError("ERROR: "
208: + ServiceUtil.getErrorMessage(outputMap));
209: } else {
210: Long numberRead = (Long) outputMap
211: .get("rowProcessed");
212: messages.add("Got " + numberRead.longValue()
213: + " entities to write to the datasource.");
214: }
215: } catch (Exception ex) {
216: return ServiceUtil
217: .returnError("ERROR parsing Entity Xml file: "
218: + ex.getMessage());
219: }
220: } else {
221: messages
222: .add("No filename/URL or complete XML document specified, doing nothing.");
223: }
224:
225: // send the notification
226: Map resp = UtilMisc.toMap("messages", messages);
227: return resp;
228: }
229:
230: public static Map entityImportDir(DispatchContext dctx, Map context) {
231: GenericValue userLogin = (GenericValue) context
232: .get("userLogin");
233: Security security = dctx.getSecurity();
234: if (!security.hasPermission("ENTITY_MAINT", userLogin)) {
235: return ServiceUtil.returnError(UtilProperties.getMessage(
236: "WebtoolsUiLabels", "WebtoolsPermissionError",
237: (Locale) context.get("locale")));
238: }
239:
240: LocalDispatcher dispatcher = dctx.getDispatcher();
241:
242: List messages = FastList.newInstance();
243:
244: String path = (String) context.get("path");
245: String mostlyInserts = (String) context.get("mostlyInserts");
246: String maintainTimeStamps = (String) context
247: .get("maintainTimeStamps");
248: String createDummyFks = (String) context.get("createDummyFks");
249: boolean deleteFiles = (String) context.get("deleteFiles") != null;
250:
251: Integer txTimeout = (Integer) context.get("txTimeout");
252: Long filePause = (Long) context.get("filePause");
253:
254: if (txTimeout == null) {
255: txTimeout = new Integer(7200);
256: }
257: if (filePause == null) {
258: filePause = new Long(0);
259: }
260:
261: if (path != null && path.length() > 0) {
262: long pauseLong = filePause != null ? filePause.longValue()
263: : 0;
264: File baseDir = new File(path);
265:
266: if (baseDir.isDirectory() && baseDir.canRead()) {
267: File[] fileArray = baseDir.listFiles();
268: FastList files = FastList.newInstance();
269: for (int a = 0; a < fileArray.length; a++) {
270: if (fileArray[a].getName().toUpperCase().endsWith(
271: "XML")) {
272: files.add(fileArray[a]);
273: }
274: }
275:
276: int passes = 0;
277: int initialListSize = files.size();
278: int lastUnprocessedFilesCount = 0;
279: FastList unprocessedFiles = FastList.newInstance();
280: while (files.size() > 0
281: && files.size() != lastUnprocessedFilesCount) {
282: lastUnprocessedFilesCount = files.size();
283: unprocessedFiles = FastList.newInstance();
284: Iterator filesItr = files.iterator();
285: while (filesItr.hasNext()) {
286: Map parseEntityXmlFileArgs = UtilMisc.toMap(
287: "mostlyInserts", mostlyInserts,
288: "createDummyFks", createDummyFks,
289: "maintainTimeStamps",
290: maintainTimeStamps, "txTimeout",
291: txTimeout, "userLogin", userLogin);
292:
293: File f = (File) filesItr.next();
294: try {
295: URL furl = f.toURI().toURL();
296: parseEntityXmlFileArgs.put("url", furl);
297: Map outputMap = dispatcher.runSync(
298: "parseEntityXmlFile",
299: parseEntityXmlFileArgs);
300: Long numberRead = (Long) outputMap
301: .get("rowProcessed");
302: messages.add("Got "
303: + numberRead.longValue()
304: + " entities from " + f);
305: if (deleteFiles) {
306: messages.add("Deleting " + f);
307: f.delete();
308: }
309: } catch (Exception e) {
310: unprocessedFiles.add(f);
311: messages
312: .add("Failed "
313: + f
314: + " adding to retry list for next pass");
315: }
316: // pause in between files
317: if (pauseLong > 0) {
318: Debug.log("Pausing for [" + pauseLong
319: + "] seconds - "
320: + UtilDateTime.nowTimestamp());
321: try {
322: Thread.sleep((pauseLong * 1000));
323: } catch (InterruptedException ie) {
324: Debug.log("Pause finished - "
325: + UtilDateTime.nowTimestamp());
326: }
327: }
328: }
329: files = unprocessedFiles;
330: passes++;
331: messages.add("Pass " + passes + " complete");
332: Debug.logInfo("Pass " + passes + " complete",
333: module);
334: }
335: lastUnprocessedFilesCount = unprocessedFiles.size();
336: messages.add("---------------------------------------");
337: messages.add("Succeeded: "
338: + (initialListSize - lastUnprocessedFilesCount)
339: + " of " + initialListSize);
340: messages.add("Failed: " + lastUnprocessedFilesCount
341: + " of " + initialListSize);
342: messages.add("---------------------------------------");
343: messages.add("Failed Files:");
344: Iterator unprocessedFilesItr = unprocessedFiles
345: .iterator();
346: while (unprocessedFilesItr.hasNext()) {
347: File file = (File) unprocessedFilesItr.next();
348: messages.add("" + file);
349: }
350: } else {
351: messages.add("path not found or can't be read");
352: }
353: } else {
354: messages.add("No path specified, doing nothing.");
355: }
356: // send the notification
357: Map resp = UtilMisc.toMap("messages", messages);
358: return resp;
359: }
360:
361: public static Map entityImportReaders(DispatchContext dctx,
362: Map context) {
363: GenericValue userLogin = (GenericValue) context
364: .get("userLogin");
365: Security security = dctx.getSecurity();
366: if (!security.hasPermission("ENTITY_MAINT", userLogin)) {
367: return ServiceUtil.returnError(UtilProperties.getMessage(
368: "WebtoolsUiLabels", "WebtoolsPermissionError",
369: (Locale) context.get("locale")));
370: }
371:
372: String readers = (String) context.get("readers");
373: String overrideDelegator = (String) context
374: .get("overrideDelegator");
375: String overrideGroup = (String) context.get("overrideGroup");
376: boolean useDummyFks = "true".equals((String) context
377: .get("createDummyFks"));
378: boolean maintainTxs = "true".equals((String) context
379: .get("maintainTimeStamps"));
380: boolean tryInserts = "true".equals((String) context
381: .get("mostlyInserts"));
382:
383: Integer txTimeoutInt = (Integer) context.get("txTimeout");
384: int txTimeout = txTimeoutInt != null ? txTimeoutInt.intValue()
385: : -1;
386:
387: List messages = FastList.newInstance();
388:
389: // parse the pass in list of readers to use
390: List readerNames = null;
391: if (UtilValidate.isNotEmpty(readers)
392: && !"none".equalsIgnoreCase(readers)) {
393: if (readers.indexOf(",") == -1) {
394: readerNames = FastList.newInstance();
395: readerNames.add(readers);
396: } else {
397: readerNames = StringUtil.split(readers, ",");
398: }
399: }
400:
401: String groupNameToUse = overrideGroup != null ? overrideGroup
402: : "org.ofbiz";
403: GenericDelegator delegator = UtilValidate
404: .isNotEmpty(overrideDelegator) ? GenericDelegator
405: .getGenericDelegator(overrideDelegator) : dctx
406: .getDelegator();
407:
408: String helperName = delegator
409: .getGroupHelperName(groupNameToUse);
410: if (helperName == null) {
411: return ServiceUtil
412: .returnError("Unable to locate the datasource helper for the group ["
413: + groupNameToUse + "]");
414: }
415:
416: // get the reader name URLs first
417: List urlList = null;
418: if (readerNames != null) {
419: urlList = EntityDataLoader.getUrlList(helperName,
420: readerNames);
421: } else if (!"none".equalsIgnoreCase(readers)) {
422: urlList = EntityDataLoader.getUrlList(helperName);
423: }
424:
425: // need a list if it is empty
426: if (urlList == null) {
427: urlList = FastList.newInstance();
428: }
429:
430: // process the list of files
431: NumberFormat changedFormat = NumberFormat.getIntegerInstance();
432: changedFormat.setMinimumIntegerDigits(5);
433: changedFormat.setGroupingUsed(false);
434:
435: List errorMessages = new LinkedList();
436: List infoMessages = new LinkedList();
437: int totalRowsChanged = 0;
438: if (urlList != null && urlList.size() > 0) {
439: messages
440: .add("=-=-=-=-=-=-= Doing a data load with the following files:");
441: Iterator urlIter = urlList.iterator();
442: while (urlIter.hasNext()) {
443: URL dataUrl = (URL) urlIter.next();
444: messages.add(dataUrl.toExternalForm());
445: }
446:
447: messages.add("=-=-=-=-=-=-= Starting the data load...");
448:
449: urlIter = urlList.iterator();
450: while (urlIter.hasNext()) {
451: URL dataUrl = (URL) urlIter.next();
452: try {
453: int rowsChanged = EntityDataLoader.loadData(
454: dataUrl, helperName, delegator,
455: errorMessages, txTimeout, useDummyFks,
456: maintainTxs, tryInserts);
457: totalRowsChanged += rowsChanged;
458: infoMessages.add(changedFormat.format(rowsChanged)
459: + " of "
460: + changedFormat.format(totalRowsChanged)
461: + " from " + dataUrl.toExternalForm());
462: } catch (GenericEntityException e) {
463: Debug.logError(e, "Error loading data file: "
464: + dataUrl.toExternalForm(), module);
465: }
466: }
467: } else {
468: messages.add("=-=-=-=-=-=-= No data load files found.");
469: }
470:
471: if (infoMessages.size() > 0) {
472: messages
473: .add("=-=-=-=-=-=-= Here is a summary of the data load:");
474: messages.addAll(infoMessages);
475: }
476:
477: if (errorMessages.size() > 0) {
478: messages
479: .add("=-=-=-=-=-=-= The following errors occured in the data load:");
480: messages.addAll(errorMessages);
481: }
482:
483: messages.add("=-=-=-=-=-=-= Finished the data load with "
484: + totalRowsChanged + " rows changed.");
485:
486: Map resultMap = ServiceUtil.returnSuccess();
487: resultMap.put("messages", messages);
488: return resultMap;
489: }
490:
491: public static Map parseEntityXmlFile(DispatchContext dctx,
492: Map context) {
493: GenericValue userLogin = (GenericValue) context
494: .get("userLogin");
495: Security security = dctx.getSecurity();
496: if (!security.hasPermission("ENTITY_MAINT", userLogin)) {
497: return ServiceUtil.returnError(UtilProperties.getMessage(
498: "WebtoolsUiLabels", "WebtoolsPermissionError",
499: (Locale) context.get("locale")));
500: }
501:
502: GenericDelegator delegator = dctx.getDelegator();
503:
504: URL url = (URL) context.get("url");
505: String xmltext = (String) context.get("xmltext");
506:
507: if (url == null && xmltext == null) {
508: return ServiceUtil
509: .returnError("No entity xml file or text specified");
510: }
511: boolean mostlyInserts = (String) context.get("mostlyInserts") != null;
512: boolean maintainTimeStamps = (String) context
513: .get("maintainTimeStamps") != null;
514: boolean createDummyFks = (String) context.get("createDummyFks") != null;
515: Integer txTimeout = (Integer) context.get("txTimeout");
516:
517: if (txTimeout == null) {
518: txTimeout = new Integer(7200);
519: }
520:
521: Long rowProcessed = new Long(0);
522: try {
523: EntitySaxReader reader = new EntitySaxReader(delegator);
524: reader.setUseTryInsertMethod(mostlyInserts);
525: reader.setMaintainTxStamps(maintainTimeStamps);
526: reader.setTransactionTimeout(txTimeout.intValue());
527: reader.setCreateDummyFks(createDummyFks);
528:
529: long numberRead = (url != null ? reader.parse(url) : reader
530: .parse(xmltext));
531: rowProcessed = new Long(numberRead);
532: } catch (Exception ex) {
533: return ServiceUtil
534: .returnError("Error parsing entity xml file: "
535: + ex.toString());
536: }
537: // send the notification
538: Map resp = UtilMisc.toMap("rowProcessed", rowProcessed);
539: return resp;
540: }
541:
542: public static Map entityExportAll(DispatchContext dctx, Map context) {
543: GenericValue userLogin = (GenericValue) context
544: .get("userLogin");
545: Security security = dctx.getSecurity();
546: if (!security.hasPermission("ENTITY_MAINT", userLogin)) {
547: return ServiceUtil.returnError(UtilProperties.getMessage(
548: "WebtoolsUiLabels", "WebtoolsPermissionError",
549: (Locale) context.get("locale")));
550: }
551:
552: GenericDelegator delegator = dctx.getDelegator();
553:
554: String outpath = (String) context.get("outpath"); // mandatory
555: Integer txTimeout = (Integer) context.get("txTimeout");
556: if (txTimeout == null) {
557: txTimeout = new Integer(7200);
558: }
559:
560: List results = new ArrayList();
561:
562: if (outpath != null && outpath.length() > 0) {
563: File outdir = new File(outpath);
564: if (!outdir.exists()) {
565: outdir.mkdir();
566: }
567: if (outdir.isDirectory() && outdir.canWrite()) {
568:
569: Iterator passedEntityNames = null;
570: try {
571: ModelReader reader = delegator.getModelReader();
572: Collection ec = reader.getEntityNames();
573: TreeSet entityNames = new TreeSet(ec);
574: passedEntityNames = entityNames.iterator();
575: } catch (Exception exc) {
576: return ServiceUtil
577: .returnError("Error retrieving entity names.");
578: }
579: int fileNumber = 1;
580:
581: while (passedEntityNames.hasNext()) {
582: long numberWritten = 0;
583: String curEntityName = (String) passedEntityNames
584: .next();
585: EntityListIterator values = null;
586:
587: try {
588: ModelEntity me = delegator
589: .getModelEntity(curEntityName);
590: if (me instanceof ModelViewEntity) {
591: results.add("[" + fileNumber + "] [vvv] "
592: + curEntityName
593: + " skipping view entity");
594: continue;
595: }
596:
597: // some databases don't support cursors, or other problems may happen, so if there is an error here log it and move on to get as much as possible
598: try {
599: values = delegator
600: .findListIteratorByCondition(
601: curEntityName, null, null,
602: null, me.getPkFieldNames(),
603: null);
604: } catch (Exception entityEx) {
605: results.add("[" + fileNumber
606: + "] [xxx] Error when writing "
607: + curEntityName + ": " + entityEx);
608: continue;
609: }
610:
611: //Don't bother writing the file if there's nothing
612: //to put into it
613: GenericValue value = (GenericValue) values
614: .next();
615: if (value != null) {
616: PrintWriter writer = new PrintWriter(
617: new BufferedWriter(
618: new OutputStreamWriter(
619: new FileOutputStream(
620: new File(
621: outdir,
622: curEntityName
623: + ".xml")),
624: "UTF-8")));
625: writer
626: .println("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
627: writer.println("<entity-engine-xml>");
628:
629: do {
630: value.writeXmlText(writer, "");
631: numberWritten++;
632: } while ((value = (GenericValue) values
633: .next()) != null);
634: writer.println("</entity-engine-xml>");
635: writer.close();
636: results.add("[" + fileNumber + "] ["
637: + numberWritten + "] "
638: + curEntityName + " wrote "
639: + numberWritten + " records");
640: } else {
641: results
642: .add("["
643: + fileNumber
644: + "] [---] "
645: + curEntityName
646: + " has no records, not writing file");
647: }
648: values.close();
649: } catch (Exception ex) {
650: if (values != null) {
651: try {
652: values.close();
653: } catch (Exception exc) {
654: //Debug.warning();
655: }
656: }
657: results.add("[" + fileNumber
658: + "] [xxx] Error when writing "
659: + curEntityName + ": " + ex);
660: }
661: fileNumber++;
662: }
663: } else {
664: results.add("Path not found or no write access.");
665: }
666: } else {
667: results.add("No path specified, doing nothing.");
668: }
669: // send the notification
670: Map resp = UtilMisc.toMap("results", results);
671: return resp;
672: }
673: }
|