From c601b75675e804f2eca668fb25c9a713716ee1ee Mon Sep 17 00:00:00 2001 From: Don Mitchell Date: Thu, 16 Jan 2014 15:41:26 -0500 Subject: [PATCH 1/4] MixedModulestore wraps most getters, update_item, delete_item with code to translate between addressing schemes based on app and persistence layer addressing scheme specification. STUD-1206 --- CHANGELOG.rst | 6 + .../contentstore/course_info_model.py | 4 +- .../contentstore/tests/test_checklists.py | 2 +- .../contentstore/tests/test_contentstore.py | 84 +---- .../contentstore/tests/test_course_updates.py | 2 +- .../contentstore/tests/test_orphan.py | 3 +- .../contentstore/tests/test_textbooks.py | 7 +- .../contentstore/tests/test_transcripts.py | 57 ++-- .../contentstore/transcripts_utils.py | 2 +- cms/djangoapps/contentstore/utils.py | 10 - .../contentstore/views/checklist.py | 5 +- cms/djangoapps/contentstore/views/course.py | 20 +- cms/djangoapps/contentstore/views/item.py | 43 ++- cms/djangoapps/contentstore/views/tabs.py | 8 +- .../models/settings/course_details.py | 39 ++- .../models/settings/course_grading.py | 28 +- .../models/settings/course_metadata.py | 3 +- .../external_auth/tests/test_shib.py | 16 +- common/djangoapps/student/tests/test_login.py | 4 +- .../xmodule/xmodule/modulestore/__init__.py | 47 +-- .../xmodule/modulestore/loc_mapper_store.py | 10 +- .../xmodule/xmodule/modulestore/locator.py | 13 + .../lib/xmodule/xmodule/modulestore/mixed.py | 310 +++++++++++++++--- .../xmodule/xmodule/modulestore/mongo/base.py | 141 ++++---- .../xmodule/modulestore/mongo/draft.py | 65 +--- .../xmodule/modulestore/split_mongo/split.py | 35 +- .../xmodule/modulestore/store_utilities.py | 16 +- .../xmodule/modulestore/tests/django_utils.py | 8 +- .../xmodule/modulestore/tests/factories.py | 8 +- .../modulestore/tests/test_location_mapper.py | 12 +- .../modulestore/tests/test_locators.py | 18 + .../tests/test_mixed_modulestore.py | 13 +- .../xmodule/modulestore/tests/test_orphan.py | 2 +- .../xmodule/modulestore/tests/test_publish.py | 6 +- .../modulestore/tests/test_split_migrator.py | 10 +- common/lib/xmodule/xmodule/modulestore/xml.py | 32 +- .../xmodule/modulestore/xml_importer.py | 41 +-- .../tests/test_submitting_problems.py | 4 +- .../tests/test_view_authentication.py | 40 ++- lms/djangoapps/courseware/views.py | 4 +- .../instructor_task/tests/test_base.py | 4 +- .../instructor_task/tests/test_integration.py | 6 +- lms/envs/acceptance.py | 1 + lms/envs/bok_choy.auth.json | 1 + lms/envs/cms/mixed_dev.py | 1 + 45 files changed, 632 insertions(+), 559 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1e464d063a1c..ea4753a51f35 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -49,6 +49,12 @@ Blades: Fix comparison of float numbers. BLD-434. Blades: Allow regexp strings as the correct answer to a string response question. BLD-475. +Common: MixedModulestore is now the only approved access to the persistence layer + - takes a new parameter 'reference_type' which can be 'Location' or 'Locator'. Mixed + then tries to ensure that every reference in any xblock gets converted to that type on + retrieval. Because we're moving to Locators, the default is Locator; so, you should change + all existing configurations to 'Location' (unless you're using split) + Common: Add feature flags to allow developer use of pure XBlocks - ALLOW_ALL_ADVANCED_COMPONENTS disables the hard-coded list of advanced components in Studio, and allows any xblock to be added as an diff --git a/cms/djangoapps/contentstore/course_info_model.py b/cms/djangoapps/contentstore/course_info_model.py index 314793db72ac..0e2551cac81f 100644 --- a/cms/djangoapps/contentstore/course_info_model.py +++ b/cms/djangoapps/contentstore/course_info_model.py @@ -102,7 +102,7 @@ def update_course_updates(location, update, passed_id=None): # update db record course_updates.data = html.tostring(course_html_parsed) - modulestore('direct').update_item(location, course_updates.data) + modulestore('direct').update_item(course_updates, 'course_info_model') return { "id": idx, @@ -158,7 +158,7 @@ def delete_course_update(location, update, passed_id): # update db record course_updates.data = html.tostring(course_html_parsed) store = modulestore('direct') - store.update_item(location, course_updates.data) + store.update_item(course_updates, 'course_info_model') return get_course_updates(location, None) diff --git a/cms/djangoapps/contentstore/tests/test_checklists.py b/cms/djangoapps/contentstore/tests/test_checklists.py index 864fe23c8333..14164680fc7a 100644 --- a/cms/djangoapps/contentstore/tests/test_checklists.py +++ b/cms/djangoapps/contentstore/tests/test_checklists.py @@ -54,7 +54,7 @@ def test_get_checklists(self): # Save the changed `checklists` to the underlying KeyValueStore before updating the modulestore self.course.save() modulestore = get_modulestore(self.course.location) - modulestore.update_metadata(self.course.location, own_metadata(self.course)) + modulestore.update_item(self.course, self.user.pk) self.assertEqual(self.get_persisted_checklists(), None) response = self.client.get(self.checklists_url) self.assertEqual(payload, response.content) diff --git a/cms/djangoapps/contentstore/tests/test_contentstore.py b/cms/djangoapps/contentstore/tests/test_contentstore.py index d28a3ab3de40..c7f4b9339c70 100644 --- a/cms/djangoapps/contentstore/tests/test_contentstore.py +++ b/cms/djangoapps/contentstore/tests/test_contentstore.py @@ -47,8 +47,6 @@ from django_comment_common.utils import are_permissions_roles_seeded from xmodule.exceptions import InvalidVersionError -import datetime -from pytz import UTC from uuid import uuid4 from pymongo import MongoClient from student.models import CourseEnrollment @@ -126,11 +124,7 @@ def check_components_on_page(self, component_types, expected_types): course.advanced_modules = component_types - # Save the data that we've just changed to the underlying - # MongoKeyValueStore before we update the mongo datastore. - course.save() - - store.update_metadata(course.location, own_metadata(course)) + store.update_item(course, self.user.username) # just pick one vertical descriptor = store.get_items(Location('i4x', 'edX', 'simple', 'vertical', None, None))[0] @@ -269,7 +263,7 @@ def test_draft_metadata(self): self.assertIn('graceperiod', own_metadata(html_module)) self.assertEqual(html_module.graceperiod, new_graceperiod) - draft_store.update_metadata(html_module.location, own_metadata(html_module)) + draft_store.update_item(html_module, self.user.username) # read back to make sure it reads as 'own-metadata' html_module = draft_store.get_item(Location('i4x', 'edX', 'simple', 'html', 'test_html', None)) @@ -385,8 +379,7 @@ def test_create_static_tab_and_rename(self): self.assertEqual(course.tabs, expected_tabs) item.display_name = 'Updated' - item.save() - module_store.update_metadata(item.location, own_metadata(item)) + module_store.update_item(item, self.user.username) course = module_store.get_item(course_location) @@ -834,9 +827,9 @@ def test_portable_link_rewrites_during_clone_course(self): html_module = module_store.get_instance(source_location.course_id, html_module_location) self.assertIsInstance(html_module.data, basestring) - new_data = html_module.data.replace('/static/', '/c4x/{0}/{1}/asset/'.format( + new_data = html_module.data = html_module.data.replace('/static/', '/c4x/{0}/{1}/asset/'.format( source_location.org, source_location.course)) - module_store.update_item(html_module_location, new_data) + module_store.update_item(html_module, None) html_module = module_store.get_instance(source_location.course_id, html_module_location) self.assertEqual(new_data, html_module.data) @@ -858,22 +851,18 @@ def test_illegal_draft_crud_ops(self): draft_store = modulestore('draft') direct_store = modulestore('direct') - CourseFactory.create(org='MITx', course='999', display_name='Robot Super Course') + course = CourseFactory.create(org='MITx', course='999', display_name='Robot Super Course') location = Location('i4x://MITx/999/chapter/neuvo') # Ensure draft mongo store does not allow us to create chapters either directly or via convert to draft self.assertRaises(InvalidVersionError, draft_store.create_and_save_xmodule, location) direct_store.create_and_save_xmodule(location) self.assertRaises(InvalidVersionError, draft_store.convert_to_draft, location) + chapter = draft_store.get_instance(course.id, location) + chapter.data = 'chapter data' - self.assertRaises(InvalidVersionError, draft_store.update_item, location, 'chapter data') - - # taking advantage of update_children and other functions never checking that the ids are valid - self.assertRaises(InvalidVersionError, draft_store.update_children, location, - ['i4x://MITx/999/problem/doesntexist']) - - self.assertRaises(InvalidVersionError, draft_store.update_metadata, location, - {'due': datetime.datetime.now(UTC)}) + with self.assertRaises(InvalidVersionError): + draft_store.update_item(chapter, 'user') self.assertRaises(InvalidVersionError, draft_store.unpublish, location) @@ -992,8 +981,8 @@ def test_export_course(self, mock_get): sequential = module_store.get_item(Location(['i4x', 'edX', 'toy', 'sequential', 'vertical_sequential', None])) private_location_no_draft = private_vertical.location.replace(revision=None) - module_store.update_children(sequential.location, sequential.children + - [private_location_no_draft.url()]) + sequential.children.append(private_location_no_draft.url()) + module_store.update_item(sequential, 'user') # read back the sequential, to make sure we have a pointer to sequential = module_store.get_item(Location(['i4x', 'edX', 'toy', @@ -1285,31 +1274,6 @@ def test_prefetch_children(self): self.assertFalse(Location(['i4x', 'edX', 'toy', 'vertical', 'vertical_test', None]) in course.system.module_data) - def test_export_course_with_unknown_metadata(self): - module_store = modulestore('direct') - content_store = contentstore() - - import_from_xml(module_store, 'common/test/data/', ['toy']) - location = CourseDescriptor.id_to_location('edX/toy/2012_Fall') - - root_dir = path(mkdtemp_clean()) - - course = module_store.get_item(location) - - metadata = own_metadata(course) - # add a bool piece of unknown metadata so we can verify we don't throw an exception - metadata['new_metadata'] = True - - # Save the data that we've just changed to the underlying - # MongoKeyValueStore before we update the mongo datastore. - course.save() - module_store.update_metadata(location, metadata) - - print 'Exporting to tempdir = {0}'.format(root_dir) - - # export out to a tempdir - export_to_xml(module_store, content_store, location, root_dir, 'test_export') - def test_export_course_without_content_store(self): module_store = modulestore('direct') content_store = contentstore() @@ -1319,16 +1283,7 @@ def test_export_course_without_content_store(self): import_from_xml(module_store, 'common/test/data/', ['toy']) location = CourseDescriptor.id_to_location('edX/toy/2012_Fall') - # Add a sequence - stub_location = Location(['i4x', 'edX', 'toy', 'sequential', 'vertical_sequential']) - sequential = module_store.get_item(stub_location) - module_store.update_children(sequential.location, sequential.children) - - # Get course and export it without a content_store - - course = module_store.get_item(location) - course.save() root_dir = path(mkdtemp_clean()) @@ -1343,7 +1298,7 @@ def test_export_course_without_content_store(self): module_store, root_dir, ['test_export_no_content_store'], draft_store=None, static_content_store=None, - target_location_namespace=course.location + target_location_namespace=location ) # Verify reimported course @@ -1810,7 +1765,8 @@ def test_metadata_inheritance(self): # crate a new module and add it as a child to a vertical module_store.create_and_save_xmodule(new_component_location) parent = verticals[0] - module_store.update_children(parent.location, parent.children + [new_component_location.url()]) + parent.children.append(new_component_location.url()) + module_store.update_item(parent, 'user') # flush the cache module_store.refresh_cached_metadata_inheritance_tree(new_component_location) @@ -1827,8 +1783,7 @@ def test_metadata_inheritance(self): # now let's define an override at the leaf node level # new_module.graceperiod = timedelta(1) - new_module.save() - module_store.update_metadata(new_module.location, own_metadata(new_module)) + module_store.update_item(new_module, self.user.username) # flush the cache and refetch module_store.refresh_cached_metadata_inheritance_tree(new_component_location) @@ -1942,10 +1897,7 @@ def test_metadata_not_persistence(self): delattr(self.video_descriptor, field_name) self.assertNotIn('html5_sources', own_metadata(self.video_descriptor)) - get_modulestore(location).update_metadata( - location, - own_metadata(self.video_descriptor) - ) + get_modulestore(location).update_item(self.video_descriptor, 'testuser') module = get_modulestore(location).get_item(location) self.assertNotIn('html5_sources', own_metadata(module)) @@ -2001,7 +1953,7 @@ def _course_factory_create_course(): Creates a course via the CourseFactory and returns the locator for it. """ course = CourseFactory.create(org='MITx', course='999', display_name='Robot Super Course') - return loc_mapper().translate_location(course.location.course_id, course.location, True, True) + return loc_mapper().translate_location(course.id, course.location, False, True) def _get_course_id(test_course_data): diff --git a/cms/djangoapps/contentstore/tests/test_course_updates.py b/cms/djangoapps/contentstore/tests/test_course_updates.py index 5ee5f1289bb7..1a112f87e14e 100644 --- a/cms/djangoapps/contentstore/tests/test_course_updates.py +++ b/cms/djangoapps/contentstore/tests/test_course_updates.py @@ -123,7 +123,7 @@ def test_no_ol_course_update(self): modulestore('direct').create_and_save_xmodule(location) course_updates = modulestore('direct').get_item(location) course_updates.data = 'bad news' - modulestore('direct').update_item(location, course_updates.data) + modulestore('direct').update_item(course_updates, 'test_course_updates') init_content = '' diff --git a/cms/djangoapps/contentstore/tests/test_orphan.py b/cms/djangoapps/contentstore/tests/test_orphan.py index e7babd25fd49..3585d81e1569 100644 --- a/cms/djangoapps/contentstore/tests/test_orphan.py +++ b/cms/djangoapps/contentstore/tests/test_orphan.py @@ -4,7 +4,6 @@ import json from contentstore.tests.utils import CourseTestCase from xmodule.modulestore.django import editable_modulestore, loc_mapper -from django.core.urlresolvers import reverse from student.models import CourseEnrollment class TestOrphan(CourseTestCase): @@ -35,7 +34,7 @@ def _create_item(self, category, name, data, metadata, parent_category, parent_n parent_location = self.course.location.replace(category=parent_category, name=parent_name) parent = editable_modulestore('direct').get_item(parent_location) parent.children.append(location.url()) - editable_modulestore('direct').update_children(parent_location, parent.children) + editable_modulestore('direct').update_item(parent, self.user.pk) def test_mongo_orphan(self): """ diff --git a/cms/djangoapps/contentstore/tests/test_textbooks.py b/cms/djangoapps/contentstore/tests/test_textbooks.py index d1fe60635440..aab4ccc39989 100644 --- a/cms/djangoapps/contentstore/tests/test_textbooks.py +++ b/cms/djangoapps/contentstore/tests/test_textbooks.py @@ -58,11 +58,8 @@ def test_view_index_xhr_content(self): } ] self.course.pdf_textbooks = content - # Save the data that we've just changed to the underlying - # MongoKeyValueStore before we update the mongo datastore. - self.course.save() store = get_modulestore(self.course.location) - store.update_metadata(self.course.location, own_metadata(self.course)) + store.update_item(self.course, self.user.pk) resp = self.client.get( self.url, @@ -200,7 +197,7 @@ def setUp(self): # MongoKeyValueStore before we update the mongo datastore. self.course.save() self.store = get_modulestore(self.course.location) - self.store.update_metadata(self.course.location, own_metadata(self.course)) + self.store.update_item(self.course, self.user.pk) self.url_nonexist = self.course_locator.url_reverse("textbooks", "20") def test_get_1(self): diff --git a/cms/djangoapps/contentstore/tests/test_transcripts.py b/cms/djangoapps/contentstore/tests/test_transcripts.py index f4c7f773adf0..57e30ed9db1e 100644 --- a/cms/djangoapps/contentstore/tests/test_transcripts.py +++ b/cms/djangoapps/contentstore/tests/test_transcripts.py @@ -63,13 +63,13 @@ def setUp(self): self.item_locator, self.item_location = self._get_locator(resp) self.assertEqual(resp.status_code, 200) + self.item = modulestore().get_item(self.item_location) # hI10vDNYz4M - valid Youtube ID with transcripts. # JMD_ifUUfsU, AKqURZnYqpk, DYpADpL7jAY - valid Youtube IDs without transcripts. - data = '