ConcurrentModificationException in Site code

Description

We have 13 similar bug reports, which started showing up only after we deployed recent changes to site from SAK-13886 and SAK-14070, so it seems this is a regression of some form.

org.sakaiproject.portal.api.PortalHandlerException: java.util.ConcurrentModificationException
at org.sakaiproject.portal.charon.SkinnableCharonPortal.doGet(SkinnableCharonPortal.java:891)
caused by: java.util.ConcurrentModificationException
at java.util.AbstractList$Itr.checkForComodification(AbstractList.java:449)
at java.util.AbstractList$Itr.next(AbstractList.java:420)
at org.sakaiproject.site.impl.BaseSitePage.<init>(BaseSitePage.java:260)
at org.sakaiproject.site.impl.BaseSite.set(BaseSite.java:578)
at org.sakaiproject.site.impl.BaseSite.<init>(BaseSite.java:224)
at org.sakaiproject.site.impl.BaseSiteService.getCachedSite(BaseSiteService.java:536)
at org.sakaiproject.site.impl.BaseSiteService.getDefinedSite(BaseSiteService.java:557)
at org.sakaiproject.site.impl.BaseSiteService.getSite(BaseSiteService.java:644)
at org.sakaiproject.site.impl.BaseSiteService.getSiteVisit(BaseSiteService.java:713)
at org.sakaiproject.site.cover.SiteService.getSiteVisit(SiteService.java:130)
at org.sakaiproject.portal.charon.handlers.ToolHandler.doTool(ToolHandler.java:141)
at org.sakaiproject.portal.charon.handlers.ToolHandler.doGet(ToolHandler.java:86)
at org.sakaiproject.portal.charon.SkinnableCharonPortal.doGet(SkinnableCharonPortal.java:891)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:690)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:269)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:188)
at org.sakaiproject.util.RequestFilter.doFilter(RequestFilter.java:592)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:215)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:188)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:213)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:174)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:117)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:108)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:174)
at org.apache.jk.server.JkCoyoteHandler.invoke(JkCoyoteHandler.java:200)
at org.apache.jk.common.HandlerRequest.invoke(HandlerRequest.java:283)
at org.apache.jk.common.ChannelSocket.invoke(ChannelSocket.java:773)
at org.apache.jk.common.ChannelSocket.processConnection(ChannelSocket.java:703)
at org.apache.jk.common.ChannelSocket$SocketConnection.runIt(ChannelSocket.java:895)
at org.apache.tomcat.util.threads.ThreadPool$ControlRunnable.run(ThreadPool.java:689)
at java.lang.Thread.run(Thread.java:595)

org.sakaiproject.portal.api.PortalHandlerException: java.util.ConcurrentModificationException
at org.sakaiproject.portal.charon.SkinnableCharonPortal.doGet(SkinnableCharonPortal.java:891)
caused by: java.util.ConcurrentModificationException
at java.util.AbstractList$Itr.checkForComodification(AbstractList.java:449)
at java.util.AbstractList$Itr.next(AbstractList.java:420)
at org.sakaiproject.site.impl.BaseSite.set(BaseSite.java:577)
at org.sakaiproject.site.impl.BaseSite.<init>(BaseSite.java:224)
at org.sakaiproject.site.impl.BaseSiteService.getCachedSite(BaseSiteService.java:536)
at org.sakaiproject.site.impl.DbSiteService$DbStorage.getSites(DbSiteService.java:789)
at org.sakaiproject.site.impl.BaseSiteService.getSites(BaseSiteService.java:1521)
at org.sakaiproject.site.cover.SiteService.getSites(SiteService.java:396)
at org.sakaiproject.portal.util.PortalSiteHelper.getAccessSites(PortalSiteHelper.java:170)
at org.sakaiproject.portal.util.PortalSiteHelper.getSubSites(PortalSiteHelper.java:187)
at org.sakaiproject.portal.charon.SkinnableCharonPortal.includeSubSites(SkinnableCharonPortal.java:371)
at org.sakaiproject.portal.charon.handlers.SiteHandler.doSite(SiteHandler.java:259)
at org.sakaiproject.portal.charon.handlers.SiteHandler.doGet(SiteHandler.java:117)
at org.sakaiproject.portal.charon.SkinnableCharonPortal.doGet(SkinnableCharonPortal.java:891)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:690)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:803)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:269)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:188)
at org.sakaiproject.util.RequestFilter.doFilter(RequestFilter.java:592)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:215)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:188)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:213)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:174)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:117)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:108)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:174)
at org.apache.jk.server.JkCoyoteHandler.invoke(JkCoyoteHandler.java:200)
at org.apache.jk.common.HandlerRequest.invoke(HandlerRequest.java:283)
at org.apache.jk.common.ChannelSocket.invoke(ChannelSocket.java:773)
at org.apache.jk.common.ChannelSocket.processConnection(ChannelSocket.java:703)
at org.apache.jk.common.ChannelSocket$SocketConnection.runIt(ChannelSocket.java:895)
at org.apache.tomcat.util.threads.ThreadPool$ControlRunnable.run(ThreadPool.java:689)
at java.lang.Thread.run(Thread.java:595)

Activity

Show:

Stephen Marquard August 3, 2008 at 10:42 PM

Result of SAK-14070.

Stephen Marquard July 31, 2008 at 3:40 AM

It appears this is a result of changes in SAK-14070. Will verify in due course based on production experience (having reverted changes from 2-5-x).

Stephen Marquard July 30, 2008 at 11:41 AM

Hypothesis: because the site object is added to the cache before the loadAll() (change in ), the loadAll() in SiteCacheImpl's notifyCachePut() sometimes co-incides with the site object being retrieved from the cache by another thread. getCachedSite() in BaseSiteService is trying to copy the site (incl. pages, groups, tools) while those are being updated by SiteCacheImpl.

Stephen Marquard July 30, 2008 at 3:27 AM

Bug history:

mysql> select BUG_ID, VERSION, REVISION, BUG_DATE, REQPATH from SAKAI_BUGS WHERE BODY LIKE '%ConcurrentModificationException%' and CAUSED_BY='java.util.ConcurrentModificationException' and body like '%BaseSite%';
----------------+------------------------------------------------------------------------------------------------+

BUG_ID

VERSION

REVISION

BUG_DATE

REQPATH

----------------+------------------------------------------------------------------------------------------------+

15771

2.5

[r49375-r5732]

2008-07-26 08:00:01

/portal/site/eb3bffff-ed84-4324-8b24-8435d442ea82

15786

2.5

[r49375-r5732]

2008-07-28 08:41:22

/portal/tool/81dd7f48-4803-4566-a46b-b0c2dddaaa2f

15796

2.5

[r49375-r5732]

2008-07-28 14:19:12

/portal/tool/89196416-5168-4eb0-8c70-c36661b182a6

15799

2.5

[r49375-r5732]

2008-07-28 15:02:32

/portal/presence/8d912f3d-d572-4419-949e-7541e49b32eb

15821

2.5

[r49375-r5732]

2008-07-29 11:00:51

/portal/tool/1ada2693-2d6a-44f0-007b-d9f404b03728/calendar

15822

2.5

[r49375-r5732]

2008-07-29 11:09:41

/portal/tool/87f34764-1f1b-4f55-80a3-74994740658d/calendar

15823

2.5

[r49375-r5732]

2008-07-29 11:57:31

/portal/tool/f6487532-504b-49c0-81c8-4fd4c665ddf5

15839

2.5

[r49375-r5732]

2008-07-29 19:28:45

/portal/tool/38310cc3-cb61-461e-ad21-62f24e4d4a70

15840

2.5

[r49375-r5732]

2008-07-29 19:28:45

/portal/tool/269ce9a1-1efd-4dc4-be6a-ba4591591497

15852

2.5

[r49375-r5732]

2008-07-30 10:16:58

/portal/tool/89ff0745-7c03-4d9d-8057-05b771eba337

15860

2.5

[r49375-r5732]

2008-07-30 11:40:29

/portal/tool/7d29d352-0400-475f-ae70-54796266c165/synMain

15861

2.5

[r49375-r5732]

2008-07-30 11:45:21

/portal/tool/12b7bd85-ae19-4cfd-b82e-f74330c509db

15862

2.5

[r49375-r5732]

2008-07-30 11:46:19

NULL

----------------+------------------------------------------------------------------------------------------------+
13 rows in set (4.99 sec)

Duplicate

Details

Priority

Affects versions

Components

Assignee

Reporter

Created July 30, 2008 at 3:24 AM
Updated October 28, 2008 at 9:52 PM
Resolved August 3, 2008 at 10:42 PM