001: /*
002: * Copyright 2005 Sun Microsystems, Inc.
003: *
004: * Licensed under the Apache License, Version 2.0 (the "License");
005: * you may not use this file except in compliance with the License.
006: * You may obtain a copy of the License at
007: *
008: * http://www.apache.org/licenses/LICENSE-2.0
009: *
010: * Unless required by applicable law or agreed to in writing, software
011: * distributed under the License is distributed on an "AS IS" BASIS,
012: * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013: * See the License for the specific language governing permissions and
014: * limitations under the License.
015: */
016:
017: package org.apache.roller.planet.business.hibernate;
018:
019: import com.sun.syndication.feed.synd.SyndEntry;
020: import com.sun.syndication.feed.synd.SyndFeed;
021: import com.sun.syndication.fetcher.FeedFetcher;
022: import com.sun.syndication.fetcher.impl.FeedFetcherCache;
023: import com.sun.syndication.fetcher.impl.HttpURLFeedFetcher;
024: import com.sun.syndication.fetcher.impl.SyndFeedInfo;
025: import java.io.File;
026: import java.net.URL;
027: import java.sql.Timestamp;
028: import java.text.MessageFormat;
029: import java.util.ArrayList;
030: import java.util.Calendar;
031: import java.util.Date;
032: import java.util.HashMap;
033: import java.util.Iterator;
034: import java.util.List;
035: import java.util.Map;
036: import java.util.Set;
037: import java.util.TreeSet;
038: import org.apache.roller.business.hibernate.*;
039: import org.hibernate.Criteria;
040: import org.hibernate.HibernateException;
041: import org.hibernate.Query;
042: import org.hibernate.Session;
043: import org.hibernate.criterion.Expression;
044: import org.hibernate.criterion.Order;
045: import org.apache.commons.logging.Log;
046: import org.apache.commons.logging.LogFactory;
047: import org.apache.roller.RollerException;
048: import org.apache.roller.planet.business.PlanetManager;
049: import org.apache.roller.planet.pojos.PlanetConfigData;
050: import org.apache.roller.planet.pojos.PlanetEntryData;
051: import org.apache.roller.planet.pojos.PlanetGroupData;
052: import org.apache.roller.planet.pojos.PlanetSubscriptionData;
053: import org.apache.roller.util.rome.DiskFeedInfoCache;
054:
055: /**
056: * Hibernate implementation of the PlanetManager.
057: */
058: public class HibernatePlanetManagerImpl implements PlanetManager {
059:
060: private static Log log = LogFactory
061: .getLog(HibernatePlanetManagerImpl.class);
062:
063: protected static final String NO_GROUP = "zzz_nogroup_zzz";
064:
065: private HibernatePersistenceStrategy strategy = null;
066: private Map lastUpdatedByGroup = new HashMap();
067:
068: public HibernatePlanetManagerImpl(HibernatePersistenceStrategy strat) {
069: this .strategy = strat;
070: }
071:
072: public void saveConfiguration(PlanetConfigData config)
073: throws RollerException {
074: strategy.store(config);
075: }
076:
077: public void saveGroup(PlanetGroupData group) throws RollerException {
078: strategy.store(group);
079: }
080:
081: public void saveEntry(PlanetEntryData entry) throws RollerException {
082: strategy.store(entry);
083: }
084:
085: public void saveSubscription(PlanetSubscriptionData sub)
086: throws RollerException {
087: PlanetSubscriptionData existing = getSubscription(sub
088: .getFeedURL());
089: if (existing == null || (existing.getId().equals(sub.getId()))) {
090: this .strategy.store(sub);
091: } else {
092: throw new RollerException(
093: "ERROR: duplicate feed URLs not allowed");
094: }
095: }
096:
097: public void deleteEntry(PlanetEntryData entry)
098: throws RollerException {
099: strategy.remove(entry);
100: }
101:
102: public void deleteGroup(PlanetGroupData group)
103: throws RollerException {
104: strategy.remove(group);
105: }
106:
107: public void deleteSubscription(PlanetSubscriptionData sub)
108: throws RollerException {
109: strategy.remove(sub);
110: }
111:
112: public PlanetConfigData getConfiguration() throws RollerException {
113: PlanetConfigData config = null;
114: try {
115: Session session = ((HibernatePersistenceStrategy) strategy)
116: .getSession();
117: Criteria criteria = session
118: .createCriteria(PlanetConfigData.class);
119: criteria.setMaxResults(1);
120: List list = criteria.list();
121: config = list.size() != 0 ? (PlanetConfigData) list.get(0)
122: : null;
123: } catch (HibernateException e) {
124: throw new RollerException(e);
125: }
126: return config;
127: }
128:
129: public PlanetSubscriptionData getSubscription(String feedURL)
130: throws RollerException {
131: try {
132: Session session = ((HibernatePersistenceStrategy) strategy)
133: .getSession();
134: Criteria criteria = session
135: .createCriteria(PlanetSubscriptionData.class);
136: criteria.setMaxResults(1);
137: criteria.add(Expression.eq("feedURL", feedURL));
138: List list = criteria.list();
139: return list.size() != 0 ? (PlanetSubscriptionData) list
140: .get(0) : null;
141: } catch (HibernateException e) {
142: throw new RollerException(e);
143: }
144: }
145:
146: public PlanetSubscriptionData getSubscriptionById(String id)
147: throws RollerException {
148: return (PlanetSubscriptionData) strategy.load(id,
149: PlanetSubscriptionData.class);
150: }
151:
152: public Iterator getAllSubscriptions() {
153: try {
154: Session session = ((HibernatePersistenceStrategy) strategy)
155: .getSession();
156: Criteria criteria = session
157: .createCriteria(PlanetSubscriptionData.class);
158: criteria.addOrder(Order.asc("feedURL"));
159: List list = criteria.list();
160: return list.iterator();
161: } catch (Throwable e) {
162: throw new RuntimeException(
163: "ERROR fetching subscription collection", e);
164: }
165: }
166:
167: public int getSubscriptionCount() throws RollerException {
168: try {
169: Session session = ((HibernatePersistenceStrategy) strategy)
170: .getSession();
171: Integer count = (Integer) session
172: .createQuery(
173: "select count(*) from org.apache.roller.planet.pojos.PlanetSubscriptionData")
174: .uniqueResult();
175: return count.intValue();
176: } catch (Throwable e) {
177: throw new RuntimeException(
178: "ERROR fetching subscription count", e);
179: }
180: }
181:
182: public synchronized List getTopSubscriptions(int offset, int length)
183: throws RollerException {
184: return getTopSubscriptions(null, offset, length);
185: }
186:
187: public synchronized List getTopSubscriptions(String groupHandle,
188: int offset, int length) throws RollerException {
189: List ret = null;
190: try {
191: Session session = ((HibernatePersistenceStrategy) strategy)
192: .getSession();
193: Query query = null;
194: if (groupHandle != null) {
195: query = session
196: .createQuery("select sub from org.apache.roller.planet.pojos.PlanetSubscriptionData sub "
197: + "join sub.groups group "
198: + "where "
199: + "group.handle=:groupHandle "
200: + "order by sub.inboundblogs desc");
201: query.setString("groupHandle", groupHandle);
202: } else {
203: query = session
204: .createQuery("select sub from org.apache.roller.planet.pojos.PlanetSubscriptionData sub "
205: + "order by sub.inboundblogs desc");
206: }
207: if (offset != 0) {
208: query.setFirstResult(offset);
209: }
210: if (length != -1) {
211: query.setMaxResults(length);
212: }
213: ret = query.list();
214: } catch (HibernateException e) {
215: throw new RollerException(e);
216: }
217: return ret;
218: }
219:
220: public PlanetGroupData getGroup(String handle)
221: throws RollerException {
222: try {
223: Session session = strategy.getSession();
224: Criteria criteria = session
225: .createCriteria(PlanetGroupData.class);
226: criteria.setMaxResults(1);
227: criteria.add(Expression.eq("handle", handle));
228: return (PlanetGroupData) criteria.uniqueResult();
229: } catch (HibernateException e) {
230: throw new RollerException(e);
231: }
232: }
233:
234: public PlanetGroupData getGroupById(String id)
235: throws RollerException {
236: return (PlanetGroupData) strategy.load(id,
237: PlanetGroupData.class);
238: }
239:
240: public List getGroups() throws RollerException {
241: try {
242: Session session = ((HibernatePersistenceStrategy) strategy)
243: .getSession();
244: Criteria criteria = session
245: .createCriteria(PlanetGroupData.class);
246: return criteria.list();
247: } catch (HibernateException e) {
248: throw new RollerException(e);
249: }
250: }
251:
252: public List getGroupHandles() throws RollerException {
253: List handles = new ArrayList();
254: Iterator list = getGroups().iterator();
255: while (list.hasNext()) {
256: PlanetGroupData group = (PlanetGroupData) list.next();
257: handles.add(group.getHandle());
258: }
259: return handles;
260: }
261:
262: public List getFeedEntries(String feedURL, int offset, int length)
263: throws RollerException {
264: // TODO: ATLAS getFeedEntries DONE
265: try {
266: Session session = ((HibernatePersistenceStrategy) strategy)
267: .getSession();
268: Criteria criteria = session
269: .createCriteria(PlanetEntryData.class);
270: criteria
271: .add(Expression.eq("subscription.feedURL", feedURL));
272: criteria.addOrder(Order.desc("pubTime"));
273: criteria.setFirstResult(offset);
274: if (length != -1)
275: criteria.setMaxResults(length);
276: return criteria.list();
277: } catch (HibernateException e) {
278: throw new RollerException(e);
279: }
280: }
281:
282: public synchronized List getAggregation(int offset, int len)
283: throws RollerException {
284: return getAggregation(null, null, null, offset, len);
285: }
286:
287: public synchronized List getAggregation(Date startDate,
288: Date endDate, int offset, int len) throws RollerException {
289: return getAggregation(null, startDate, endDate, offset, len);
290: }
291:
292: public synchronized List getAggregation(PlanetGroupData group,
293: int offset, int len) throws RollerException {
294: return getAggregation(group, null, null, offset, len);
295: }
296:
297: public synchronized List getAggregation(PlanetGroupData group,
298: Date startDate, Date endDate, int offset, int length)
299: throws RollerException {
300: // TODO: ATLAS getAggregation DONE TESTED
301: List ret = null;
302: if (endDate == null)
303: endDate = new Date();
304: try {
305: String groupHandle = (group == null) ? NO_GROUP : group
306: .getHandle();
307: long startTime = System.currentTimeMillis();
308: Session session = ((HibernatePersistenceStrategy) strategy)
309: .getSession();
310:
311: if (group != null) {
312: StringBuffer sb = new StringBuffer();
313: sb
314: .append("select e from org.apache.roller.planet.pojos.PlanetEntryData e ");
315: sb.append("join e.subscription.groups g ");
316: sb
317: .append("where g.handle=:groupHandle and e.pubTime < :endDate ");
318: if (startDate != null) {
319: sb.append("and e.pubTime > :startDate ");
320: }
321: sb.append("order by e.pubTime desc");
322: Query query = session.createQuery(sb.toString());
323: query.setParameter("groupHandle", group.getHandle());
324: query.setFirstResult(offset);
325: if (length != -1)
326: query.setMaxResults(length);
327: query.setParameter("endDate", endDate);
328: if (startDate != null) {
329: query.setParameter("startDate", startDate);
330: }
331: ret = query.list();
332: } else {
333: StringBuffer sb = new StringBuffer();
334: sb
335: .append("select e from org.apache.roller.planet.pojos.PlanetEntryData e ");
336: sb.append("join e.subscription.groups g ");
337: sb
338: .append("where (g.handle='external' or g.handle='all') ");
339: sb.append("and e.pubTime < :endDate ");
340: if (startDate != null) {
341: sb.append("and e.pubTime > :startDate ");
342: }
343: sb.append("order by e.pubTime desc");
344: Query query = session.createQuery(sb.toString());
345: query.setFirstResult(offset);
346: if (length != -1)
347: query.setMaxResults(length);
348: query.setParameter("endDate", endDate);
349: if (startDate != null) {
350: query.setParameter("startDate", startDate);
351: }
352: ret = query.list();
353: }
354: Date retLastUpdated = null;
355: if (ret.size() > 0) {
356: PlanetEntryData entry = (PlanetEntryData) ret.get(0);
357: retLastUpdated = entry.getPubTime();
358: } else {
359: retLastUpdated = new Date();
360: }
361: lastUpdatedByGroup.put(groupHandle, retLastUpdated);
362:
363: long endTime = System.currentTimeMillis();
364: log.debug("Generated aggregation in "
365: + ((endTime - startTime) / 1000.0) + " seconds");
366:
367: } catch (Throwable e) {
368: log.error("ERROR: building aggregation for: "
369: + group.getHandle(), e);
370: throw new RollerException(e);
371: }
372: return ret;
373: }
374:
375: public synchronized void clearCachedAggregations() {
376: lastUpdatedByGroup.clear();
377: }
378:
379: public Date getLastUpdated() {
380: return (Date) lastUpdatedByGroup.get(NO_GROUP);
381: }
382:
383: public Date getLastUpdated(PlanetGroupData group) {
384: return (Date) lastUpdatedByGroup.get(group);
385: }
386:
387: public void refreshEntries(String cacheDirPath)
388: throws RollerException {
389:
390: Date now = new Date();
391: long startTime = System.currentTimeMillis();
392: PlanetConfigData config = getConfiguration();
393:
394: // can't continue without cache dir
395: if (cacheDirPath == null) {
396: log
397: .warn("Planet cache directory not set, aborting refresh");
398: return;
399: }
400:
401: // allow ${user.home} in cache dir property
402: String cacheDirName = cacheDirPath.replaceFirst(
403: "\\$\\{user.home}", System.getProperty("user.home"));
404:
405: // allow ${catalina.home} in cache dir property
406: if (System.getProperty("catalina.home") != null) {
407: cacheDirName = cacheDirName.replaceFirst(
408: "\\$\\{catalina.home}", System
409: .getProperty("catalina.home"));
410: }
411:
412: // create cache dir if it does not exist
413: File cacheDir = null;
414: try {
415: cacheDir = new File(cacheDirName);
416: if (!cacheDir.exists())
417: cacheDir.mkdirs();
418: } catch (Exception e) {
419: log.error("Unable to create planet cache directory");
420: return;
421: }
422:
423: // abort if cache dir is not writable
424: if (!cacheDir.canWrite()) {
425: log.error("Planet cache directory is not writable");
426: return;
427: }
428:
429: FeedFetcherCache feedInfoCache = new DiskFeedInfoCache(
430: cacheDirName);
431:
432: if (config.getProxyHost() != null && config.getProxyPort() > 0) {
433: System.setProperty("proxySet", "true");
434: System.setProperty("http.proxyHost", config.getProxyHost());
435: System.setProperty("http.proxyPort", Integer
436: .toString(config.getProxyPort()));
437: }
438: /** a hack to set 15 sec timeouts for java.net.HttpURLConnection */
439: System.setProperty("sun.net.client.defaultConnectTimeout",
440: "15000");
441: System
442: .setProperty("sun.net.client.defaultReadTimeout",
443: "15000");
444:
445: FeedFetcher feedFetcher = new HttpURLFeedFetcher(feedInfoCache);
446: //FeedFetcher feedFetcher = new HttpClientFeedFetcher(feedInfoCache);
447: feedFetcher.setUsingDeltaEncoding(false);
448: feedFetcher.setUserAgent("RollerPlanetAggregator");
449:
450: // Loop through all subscriptions in the system
451: Iterator subs = getAllSubscriptions();
452: while (subs.hasNext()) {
453:
454: long subStartTime = System.currentTimeMillis();
455:
456: PlanetSubscriptionData sub = (PlanetSubscriptionData) subs
457: .next();
458:
459: // reattach sub. sub gets detached as we iterate
460: sub = this .getSubscriptionById(sub.getId());
461:
462: Set newEntries = this .getNewEntries(sub, feedFetcher,
463: feedInfoCache);
464: int count = newEntries.size();
465:
466: log.debug(" Entry count: " + count);
467: if (count > 0) {
468: sub.purgeEntries();
469: sub.addEntries(newEntries);
470: this .saveSubscription(sub);
471: this .strategy.flush();
472: }
473: long subEndTime = System.currentTimeMillis();
474: log.debug(" " + count + " - "
475: + ((subEndTime - subStartTime) / 1000.0)
476: + " seconds to process (" + count + ") entries of "
477: + sub.getFeedURL());
478: }
479: // Clear the aggregation cache
480: clearCachedAggregations();
481:
482: long endTime = System.currentTimeMillis();
483: log.info("--- DONE --- Refreshed entries in "
484: + ((endTime - startTime) / 1000.0) + " seconds");
485: }
486:
487: protected Set getNewEntries(PlanetSubscriptionData sub,
488: FeedFetcher feedFetcher, FeedFetcherCache feedInfoCache)
489: throws RollerException {
490:
491: Set newEntries = new TreeSet();
492: SyndFeed feed = null;
493: URL feedURL = null;
494: Date lastUpdated = new Date();
495: try {
496: feedURL = new URL(sub.getFeedURL());
497: log.debug("Get feed from cache " + sub.getFeedURL());
498: feed = feedFetcher.retrieveFeed(feedURL);
499: SyndFeedInfo feedInfo = feedInfoCache.getFeedInfo(feedURL);
500: if (feedInfo.getLastModified() != null) {
501: long lastUpdatedLong = ((Long) feedInfo
502: .getLastModified()).longValue();
503: if (lastUpdatedLong != 0) {
504: lastUpdated = new Date(lastUpdatedLong);
505: }
506: }
507: Thread.sleep(100); // be nice
508: } catch (Exception e) {
509: log.warn("ERROR parsing " + sub.getFeedURL() + " : "
510: + e.getClass().getName() + " : " + e.getMessage());
511: log.debug(e);
512: return newEntries; // bail out
513: }
514: if (lastUpdated != null && sub.getLastUpdated() != null) {
515: Calendar feedCal = Calendar.getInstance();
516: feedCal.setTime(lastUpdated);
517:
518: Calendar subCal = Calendar.getInstance();
519: subCal.setTime(sub.getLastUpdated());
520:
521: if (!feedCal.after(subCal)) {
522: if (log.isDebugEnabled()) {
523: String msg = MessageFormat
524: .format(" Skipping ({0} / {1})",
525: new Object[] { lastUpdated,
526: sub.getLastUpdated() });
527: log.debug(msg);
528: }
529: return newEntries; // bail out
530: }
531: }
532: if (feed.getPublishedDate() != null) {
533: sub.setLastUpdated(feed.getPublishedDate());
534: // saving sub here causes detachment issues, so we save it later
535: }
536:
537: // Horrible kludge for Feeds without entry dates: most recent entry is
538: // given feed's last publish date (or yesterday if none exists) and
539: // earler entries are placed at once day intervals before that.
540: Calendar cal = Calendar.getInstance();
541: if (sub.getLastUpdated() != null) {
542: cal.setTime(sub.getLastUpdated());
543: } else {
544: cal.setTime(new Date());
545: cal.add(Calendar.DATE, -1);
546: }
547:
548: // Populate subscription object with new entries
549: Iterator entries = feed.getEntries().iterator();
550: while (entries.hasNext()) {
551: try {
552: SyndEntry romeEntry = (SyndEntry) entries.next();
553: PlanetEntryData entry = new PlanetEntryData(feed,
554: romeEntry, sub);
555: log.debug("Entry title=" + entry.getTitle()
556: + " content size="
557: + entry.getContent().length());
558: if (entry.getPubTime() == null) {
559: log
560: .debug("No published date, assigning fake date for "
561: + feedURL);
562: entry.setPubTime(new Timestamp(cal
563: .getTimeInMillis()));
564: }
565: if (entry.getPermalink() == null) {
566: log.warn("No permalink, rejecting entry from "
567: + feedURL);
568: } else {
569: newEntries.add(entry);
570: }
571: cal.add(Calendar.DATE, -1);
572: } catch (Exception e) {
573: log.error("ERROR processing subscription entry", e);
574: }
575: }
576: return newEntries;
577: }
578:
579: }
|