Setting up an Azure Data Lake and Azure Data Factory using Powershell

Login-AzureRmAccount

#first ensure that you have an Azure Data Lake that you want to use for ODX
#$resourceGroups = Get-AzureRmResourceGroup
#$azureDataLakeNames = “”;

# foreach ($resourceGroup in $resourceGroups) {
# $azureDataLake = Get-AzureRmDataLakeStoreAccount -ResourceGroupName $resourceGroup.ResourceGroupName
#$azureDataLake
# $azureDataLakeName = $azureDataLake.Name
# $azureDataLakeNameLength = $azureDataLakeName.Length
# $azureDataLakeNameLength -gt 0
# if ($azureDataLakeNameLength -gt 0) {
# $azureDataLakeNames += ” ” + $azureDataLake.Name + ” (resource group: ” + $resourceGroup.ResourceGroupName + ” & location: ” + $resourceGroup.Location + “)”
# }
# }
# “———–”
#”DataLakeNames: ” + $azureDataLakeNames
#—————————————————————————————-
#—————————————————————————————-
#REQUIRED: you must enter a unique appname which will be used as the security principal
$appname = “sune”
#OPTIONAL: change the password for the security principal password
$password = “Xyzpdq”
#run the above script, and replace DATALAKESTORENAME with the appropriate name/rg/location from your existing data lake store; or enter a new name to have a data lake created
$dataLakeStoreName = “sunelake”
$odxResourceGroup = “odxDemo”
$dataLakeLocation = “Central US” #Central US, East US 2, North Europe
#recommended to use the same resource group as the data factory for simplicity, but you can use any resource group or enter a new name to create
$dataFactoryResourceGroup = $dataLakeStoreResourceGroup
#specify where you want your data factory – current options are East US, North Europe, West Central US, and West US
$dataFactoryLocation = “West US”
#—————————————————————————————-
#—————————————————————————————-

#create odxResourceGroup, if it does not exist
Get-AzureRmResourceGroup -Name $odxResourceGroup -ErrorVariable notPresent1 -ErrorAction 0
if ($notPresent1)
{
New-AzureRmResourceGroup -Location $dataLakeLocation -Name $odxResourceGroup
}

#create data lake, if it does not exist
Get-AzureRmDataLakeStoreAccount -Name $dataLakeStoreName -ErrorVariable notPresent2 -ErrorAction 0
if ($notPresent2)
{
New-AzureRmDataLakeStoreAccount -Location $dataLakeLocation -Name $dataLakeStoreName -ResourceGroupName $odxResourceGroup
}

$homepage = “https://ODXPS.com/” + $appname

#create security principal, if it does not exist
$app = New-AzureRmADApplication -DisplayName $appname -HomePage $homepage -IdentifierUris $homepage -Password $password
$app = Get-AzureRmADApplication -DisplayName $appname

$servicePrincipal = New-AzureRmADServicePrincipal -ApplicationId $app.ApplicationId
Start-Sleep 10
New-AzureRmRoleAssignment -RoleDefinitionName “Contributor” -Id $servicePrincipal.Id -ResourceGroupName $odxResourceGroup
New-AzureRmRoleAssignment -RoleDefinitionName “Data Factory Contributor” -Id $servicePrincipal.Id -ResourceGroupName $odxResourceGroup
New-AzureRmRoleAssignment -RoleDefinitionName “Reader” -Id $servicePrincipal.Id -ResourceGroupName $odxResourceGroup

#Set-AzureRmDataLakeStoreItemAclEntry -AccountName $dataLakeStoreName -Path / -AceType User -Id $app.ApplicationId -Permissions All
Set-AzureRmDataLakeStoreItemAclEntry -AccountName $dataLakeStoreName -Path / -AceType User -Id $servicePrincipal.Id -Permissions All
Get-AzureRmDataLakeStoreItem -Account $dataLakeStoreName -Path /ODX -ErrorVariable notPresent3 -ErrorAction 0
if ($notPresent3)
{
New-AzureRmDataLakeStoreItem -Folder -AccountName $dataLakeStoreName -Path /ODX
}
Set-AzureRmDataLakeStoreItemAclEntry -AccountName $dataLakeStoreName -Path /ODX -AceType User -Id $servicePrincipal.Id -Permissions All
#Start-Sleep 60 #there seems to be a lag between when these permissions are added and when they are applied…trying 1 minutes to start

$subscription = Get-AzureRmSubscription
$subscriptionId= ($subscription).Id
$tenantId = ($subscription).TenantId

#ensure there are permissions
#Get-AzureRmDataLakeStoreItemAclEntry -Account $dataLakeStoreName -Path /

#get information on datalake
$dataLake = Get-AzureRmDataLakeStoreAccount -Name $dataLakeStoreName

#here is a printout
“—————————————————————”
“—————————————————————”
$text1= “Azure Data Lake Name: ” + $dataLakeStoreName + “`r`n” +
“Tenant ID: ” + $tenantId + “`r`n” +
“Client ID: ” + $app.ApplicationId + “`r`n” +
“Client Secret: ” + $password + “`r`n” +
“Subscription ID: ” + $subscriptionId + “`r`n” +
“Resource Group Name: ” + $odxResourceGroup + “`r`n” +
“Data Lake URL: adl://” + $dataLake.Endpoint + “`r`n” +
“Location: ” + $dataFactoryLocation
“—————————————————————”
“—————————————————————”

Out-File C:\Users\MattDyor\Desktop\DataLake.ps1 -InputObject $text1

Leave a Reply

Your email address will not be published. Required fields are marked *