Commit 58822245 authored by Mike Ryan's avatar Mike Ryan

Initial commit

parents
MIT License
Copyright (c) 2017 Mike Ryan
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
# d8-migrate-example-002
This module, implemented on behalf of [Acquia](https://www.acquia.com/) for their (fictionalized) client Acme, demonstrates the following Drupal 8 migration techniques:
1. Configuring migration though the admin UI.
2. Migration from CSV files.
3. Migration from a JSON feed.
4. Migration from a SOAP feed.
5. Migration from an OAuth-authenticated feed.
See http://virtuoso-performance.com/blog-post-link for more information.
Migration from various content sources to a new Drupal 8 instance.
To configure the migrations:
1. Go to /admin/structure/migrate/acme_migrate.
2. Upload the blog CSV file.
3. Enter the endpoint and credentials for the CF service.
4. Enter the endpoint and credentials for the K service.
To manage the migration process:
# migrate-status command
docroot$ drush ms
Group: acme Status Total Imported Unprocessed Last imported
blog Idle 159 0 159
doctor Idle 2313 0 2313
event Idle 250 0 250
location Idle 148 0 148
service Idle 27 0 27
# migrate-import command - run all migrations
docroot$ drush mi --all
Processed 159 items (159 created, 0 updated, 0 failed, 0 ignored) - done with 'blog'
Processed 148 items (148 created, 0 updated, 0 failed, 0 ignored) - done with 'location'
...
# Run one migration
docroot$ drush mi location
Processed 148 items (148 created, 0 updated, 0 failed, 0 ignored) - done with 'location'
# Run a small sample for testing
docroot$ drush mi location --limit=10
Processed 10 items (10 created, 0 updated, 0 failed, 0 ignored) - done with 'location'
# Migrate a single item for testing (with source ID 4)
docroot$ drush mi location --idlist=4
Processed 1 item (1 created, 0 updated, 0 failed, 0 ignored) - done with 'location'
# migrate-rollback command
drush mr --all
Rolled back 148 items - done with 'location'
...
drush mr location
Rolled back 148 items - done with 'location'
# Check for error messages
docroot$ drush mmsg location
No messages for this migration
name: Migration for Acme
type: module
description: Migrate content from various sources to Drupal 8
core: 8.x
package: Acme
dependencies:
- migrate_plus
- migrate_source_csv
acme_migrate.add_group_action:
route_name: acme_migrate.configuration
title: 'Configure Acme migration'
appears_on:
- entity.migration_group.list
configure acme migration:
title: 'Configure Acme migration'
acme_migrate.configuration:
path: '/admin/structure/migrate/acme_migrate'
defaults:
_title: 'Configure Acme migration'
_form: '\Drupal\acme_migrate\Form\MigrationConfigurationForm'
requirements:
_permission: 'configure acme migration'
services:
acme_migrate.geofield_configurator:
class: Drupal\acme_migrate\EventSubscriber\GeoFieldConfigurator
tags:
- { name: event_subscriber }
acme_migrate.prepare_row:
class: Drupal\acme_migrate\EventSubscriber\PrepareRow
tags:
- { name: event_subscriber }
acme_migrate.redirects:
class: Drupal\acme_migrate\EventSubscriber\Redirects
tags:
- { name: event_subscriber }
acme_migrate.update_event_filter:
class: Drupal\acme_migrate\EventSubscriber\UpdateEventFilter
tags:
- { name: event_subscriber }
id: blog
migration_group: acme
migration_tags: {}
label: 'Blog nodes'
source:
plugin: csv
header_row_count: 1
keys:
- postID
process:
title: postName
uid:
# JRoe-updated posts will be assigned to her D8 account (uid 66).
plugin: static_map
source: lastUpdatedBy
map:
JRoe: 66
# All others will go under the admin account.
default_value: 1
langcode:
plugin: default_value
default_value: en
status:
plugin: default_value
default_value: 1
created:
plugin: callback
callable: strtotime
source: dateStamp
changed: '@created'
'body/value': content
'body/format':
plugin: default_value
default_value: full_html
# Serialized metatags array constructed in prepareRow().
field_metatags: metatags
field_topics:
-
plugin: explode
source: categories
delimiter: ,
-
plugin: entity_generate
# Entity reference to (if necessary) new node generated in prepareRow().
field_author: author
field_main_image: main_image
destination:
plugin: 'entity:node'
default_bundle: blog
id: doctor
migration_group: acme
label: 'Doctor nodes'
source:
plugin: url
# We want to reimport any doctors whose source data has changed.
track_changes: true
# Counting the available records requires fetching the whole feed - cache the
# counts to minimize overhead.
cache_counts: true
data_fetcher_plugin: http
data_parser_plugin: json
item_selector: /providers
# Note that the endpoint and actual credentials are not provided here, but are
# merged in via the configuration form.
authentication:
plugin: oauth2
grant_type: client_credentials
token_url: /oauth2/token
ids:
id:
type: integer
fields:
-
name: id
label: Provider ID
selector: id
-
name: full_name
label: Full Name
selector: name/full_name
-
name: last_updated
label: Last updated timestamp
selector: metadata/last_updated
-
name: image_url
label: Image URL
selector: image_url
-
name: specialties
label: Specialty data
selector: specialties
-
name: external_id
label: External ID
selector: external_id
constants:
picture_directory: public://profile_images/
process:
title: full_name
status:
plugin: default_value
default_value: 1
uid:
plugin: default_value
default_value: 1
created: last_updated
changed: last_updated
field_cactus_id: external_id
field_doctor_specialty:
plugin: entity_generate
source: specialty
field_doctor_subspecialty:
plugin: entity_generate
source: subspecialty
field_k_id: id
field_k_url: k_url
# From https://www.acme.com/profile_images/9629.jpg, generate a
# destination_basename of 9629.jpg...
destination_basename:
plugin: callback
callable: basename
source: image_url
# ...and on to a destination_path of public://profile_images/9629.jpg.
destination_path:
plugin: concat
source:
- 'constants/picture_directory'
- '@destination_basename'
field_picture:
-
plugin: skip_on_empty
method: process
source: image_url
-
plugin: file_copy
source:
- image_url
- '@destination_path'
-
plugin: entity_generate
destination:
plugin: 'entity:node'
default_bundle: doctor
id: event
migration_group: acme
label: 'Event nodes'
source:
plugin: url
# To remigrate any changed events.
track_changes: true
data_fetcher_plugin: http # Ignored - SoapClient does the fetching.
data_parser_plugin: soap
function: GetClientSessionsByClientId
item_selector: SessionBOLExternal
response_type: object
ids:
ClassID:
type: integer
SessionID:
type: integer
SectionID:
type: integer
fields:
-
name: ClientSessionID
label: ClientSessionID
selector: ClientSessionID
-
name: SessionID
label: SessionID
selector: SessionID
-
name: ClassID
label: ClassID
selector: ClassID
-
name: SectionID
label: SectionID
selector: SectionID
-
name: SessionLocation
label: SessionLocation
selector: SessionLocation
-
name: SessionRoom
label: SessionRoom
selector: SessionRoom
-
name: StartDateTime
label: StartDateTime
selector: StartDateTime
-
name: EndDateTime
label: EndDateTime
selector: EndDateTime
-
name: StartRegistrationDateTime
label: StartRegistrationDateTime
selector: StartRegistrationDateTime
-
name: EndRegistrationDateTime
label: EndRegistrationDateTime
selector: EndRegistrationDateTime
-
name: Days
label: Days
selector: Days
-
name: TotalSeats
label: TotalSeats
selector: TotalSeats
-
name: FilledSeats
label: FilledSeats
selector: FilledSeats
-
name: InstructorName
label: InstructorName
selector: InstructorName
-
name: SessionFee
label: SessionFee
selector: SessionFee
-
name: SessionNotes
label: SessionNotes
selector: SessionNotes
-
name: PublicTransportationAccess
label: PublicTransportationAccess
selector: PublicTransportationAccess
-
name: HandicappedAccess
label: HandicappedAccess
selector: HandicappedAccess
-
name: ParkingNotes
label: ParkingNotes
selector: ParkingNotes
-
name: FeeNotes
label: FeeNotes
selector: FeeNotes
-
name: SupplyNotes
label: SupplyNotes
selector: SupplyNotes
-
name: AcceptsAmericanExpress
label: AcceptsAmericanExpress
selector: AcceptsAmericanExpress
-
name: AcceptsVISA
label: AcceptsVISA
selector: AcceptsVISA
-
name: AcceptsMasterCard
label: AcceptsMasterCard
selector: AcceptsMasterCard
-
name: AcceptsDiscover
label: AcceptsDiscover
selector: AcceptsDiscover
process:
title: ClassName
status:
plugin: default_value
default_value: 1
uid:
plugin: default_value
default_value: 1
'field_address/country_code':
plugin: default_value
default_value: US
'field_address/langcode':
plugin: default_value
default_value: en
'field_address/address_line1': AddressLineOne
'field_address/address_line2': AddressLineTwo
'field_address/locality': City
# This has the form US-IL.
'field_address/administrative_area':
plugin: concat
delimiter: -
source:
- '@field_address/country_code'
- State
'field_address/postal_code': ZipCode
'field_description/value': ClassDescription
'field_description/format':
plugin: default_value
default_value: full_html
field_short_description: ShortClassDescription
field_start_date: StartDateTime
field_end_date: EndDateTime
field_instructor: InstructorName
field_location_name:
plugin: entity_generate
source: BusinessName
field_registration_price: SessionFee
field_remaining_spots: RemainingSeats
field_service:
plugin: entity_generate
source: ServiceLineName
field_synchronized_title: ClassName
destination:
plugin: 'entity:node'
default_bundle: event
overwrite_properties:
- 'field_address/address_line1'
- 'field_address/address_line2'
- 'field_address/locality'
- 'field_address/administrative_area'
- 'field_address/postal_code'
- field_start_date
- field_end_date
- field_instructor
- field_location_name
- field_registration_price
- field_remaining_spots
- field_synchronized_title
id: location
migration_group: acme
migration_tags: {}
label: 'Location nodes'
source:
plugin: csv
path: modules/custom/acme_migrate/data/locations_data.csv
header_row_count: 1
keys:
- itemValue
process:
title: locationName
uid:
plugin: default_value
default_value: 1
langcode:
plugin: default_value
default_value: en
status: active
'field_description/value': description
'field_description/format':
plugin: default_value
default_value: full_html
'field_address/country_code':
plugin: default_value
default_value: US
'field_address/langcode':
plugin: default_value
default_value: en
'field_address/address_line1': streetAddress1
'field_address/address_line2': streetAddress2
'field_address/locality': city
# This has the form US-IL.
'field_address/administrative_area':
plugin: concat
delimiter: -
source:
- '@field_address/country_code'
- state
'field_address/postal_code': zip
field_geofield:
plugin: geofield_latlon
source:
- latitude
- longitude
field_location_type:
-
plugin: static_map
source: categoryName
bypass: true
map:
'': Other
-
plugin: entity_generate
field_phone_number: phoneNumber
destination:
plugin: 'entity:node'
default_bundle: location
id: service
migration_group: acme
migration_tags: {}
label: 'Service nodes'
source:
plugin: csv
path: modules/custom/acme_migrate/data/services.csv
header_row_count: 1
keys:
- serviceID
process:
title: serviceName
uid:
plugin: default_value
default_value: 1
langcode:
plugin: default_value
default_value: en
status: active
'field_description/value': serviceDescription
'field_description/format':
plugin: default_value
default_value: full_html
destination:
plugin: 'entity:node'
default_bundle: service
id: acme
label: Acme imports
description: Migrations importing into the Drupal 8 Acme site
dependencies:
enforced:
module:
- acme_migrate
categoryID,categoryName,categoryName2,locationName,description,streetAddress,city,state,zip,phoneNumber,latitude,longitude,Website,alternateURL,mobileSwipeIcon,mobileDetailPageCMSRecordID,mobileGalleryViewOverlayText,fullSiteImage,fullsiteOverwriteLocationText,itemValue
0,NULL,,A Location,,123 Main St.,Anytown,AB,12345,555-555-5555,40.123456,-90.789012,http://www.acme.com/s/a-location/,,,,,,,106
categoryID,categoryName,serviceID,serviceName,serviceDescription,overwriteURL
1,A Service Category,46,A Service Name,"Full description of services.",/s/a-service-name/
<?php
namespace Drupal\acme_migrate\EventSubscriber;
use Drupal\migrate\Event\MigrateEvents;
use Drupal\migrate\Event\MigrateImportEvent;
use Symfony\Component\EventDispatcher\EventSubscriberInterface;
/**
* Handle disabling automatic geocoding during location migration.
*/
class GeoFieldConfigurator implements EventSubscriberInterface {
/**
* The geocoder settings before we changed them.
*
* @var array
*/
protected $originalSettings;
/**
* {@inheritdoc}
*/
public static function getSubscribedEvents() {
$events[MigrateEvents::PRE_IMPORT] = 'onMigrationPreImport';
$events[MigrateEvents::POST_IMPORT] = 'onMigrationPostImport';
return $events;
}
/**
* Disable automatic geocoding for field_geofield.
*
* @param \Drupal\migrate\Event\MigrateImportEvent $event
* The import event.
*/
public function onMigrationPreImport(MigrateImportEvent $event) {
if ($event->getMigration()->id() == 'location') {
$fields = \Drupal::entityTypeManager()->getStorage('field_config')->loadByProperties(['field_name' => 'field_geofield']);
if ($fields) {
/** @var \Drupal\field\Entity\FieldConfig $field */
if ($field = $fields['node.location.field_geofield']) {
$this->originalSettings = $field->getThirdPartySettings('geocoder_field');
$field->setThirdPartySetting('geocoder_field', 'method', 'none');
$field->save();
}
}
}
}
/**
* Re-enable automatic geocoding for field_geofield.
*
* @param \Drupal\migrate\Event\MigrateImportEvent $event
* The import event.
*/
public function onMigrationPostImport(MigrateImportEvent $event) {
if ($event->getMigration()->id() == 'location') {
$fields = \Drupal::entityTypeManager()->getStorage('field_config')->loadByProperties(['field_name' => 'field_geofield']);
if ($fields) {
/** @var \Drupal\field\Entity\FieldConfig $field */
if ($field = $fields['node.location.field_geofield']) {
foreach ($this->originalSettings as $key => $value) {
$field->setThirdPartySetting('geocoder_field', $key, $value);
}
$field->save();
}
}
}
}
}
This diff is collapsed.
<?php
namespace Drupal\acme_migrate\EventSubscriber;
use Drupal\migrate\Event\MigrateEvents;
use Drupal\migrate\Event\MigratePostRowSaveEvent;
use Drupal\redirect\Entity\Redirect;
use Symfony\Component\EventDispatcher\EventSubscriberInterface;
/**
* Add redirects.
*/
class Redirects implements EventSubscriberInterface {
/**
* {@inheritdoc}
*/
public static function getSubscribedEvents() {
$events[MigrateEvents::POST_ROW_SAVE] = 'onMigrationPostRowSave';
return $events;
}
/**
* Add redirects for location nodes.
*
* @param \Drupal\migrate\Event\MigratePostRowSaveEvent $event
* The post-save event.
*/
public function onMigrationPostRowSave(MigratePostRowSaveEvent $event) {
$redirects = [];
switch ($event->getMigration()->id()) {
case 'location':
$source_id = $event->getRow()->getSourceProperty('itemValue');
$destination_id = $event->getDestinationIdValues();
$redirect_properties = [
'redirect_redirect' => 'entity:node/' . reset($destination_id),
'status_code' => 301,
'uid' => $event->getRow()->getDestinationProperty('uid'),
];
// Handle the legacy detail link.
$redirects[] = $redirect_properties + [
'redirect_source' => [
'path' => 'locations/locationDetail.aspx',
'query' => ['id' => $source_id],
],
];
// If there's a local or acme.com website link, redirect it.
$urls = array_filter([
$event->getRow()->getSourceProperty('Website'),
$event->getRow()->getSourceProperty('alternateUrl'),
]);
foreach ($urls as $url) {
$url = preg_replace('|http://(www.)?acme.com/|', '', $url);
// Ignore other websites.
if (substr($url, 0, 7) != 'http://') {
$redirects[] = $redirect_properties + [
'redirect_source' => [
'path' => $url,
'query' => [],
],
];
}
}
break;
case 'service':