Hadoop HDFS Cluster Service Metrics Property Bag Data Provider

Ambari.SCOM.Module.HdfsServiceMetricsDataSource (DataSourceModuleType)

This module collects all metrics available for target HDFS Cluster Service into property bag.

Element properties:

TypeDataSourceModuleType
IsolationAny
AccessibilityInternal
RunAsDefault
OutputTypeSystem.PropertyBagData

Member Modules:

ID Module Type TypeId RunAs 
DS DataSource Ambari.SCOM.Module.ClusterServiceMetricsDataSource Default

Overrideable Parameters:

IDParameterTypeSelectorDisplay NameDescription
IntervalSecondsint$Config/IntervalSeconds$Interval (sec)
SyncTimestring$Config/SyncTime$Sync Time
TimeoutSecondsint$Config/TimeoutSeconds$Timeout (sec)

Source Code:

<DataSourceModuleType ID="Ambari.SCOM.Module.HdfsServiceMetricsDataSource" Accessibility="Internal">
<Configuration>
<xsd:element xmlns:xsd="http://www.w3.org/2001/XMLSchema" name="IntervalSeconds" type="xsd:integer"/>
<xsd:element xmlns:xsd="http://www.w3.org/2001/XMLSchema" minOccurs="0" maxOccurs="1" name="SyncTime" type="xsd:string"/>
<xsd:element xmlns:xsd="http://www.w3.org/2001/XMLSchema" name="TimeoutSeconds" type="xsd:int"/>
</Configuration>
<OverrideableParameters>
<OverrideableParameter ID="IntervalSeconds" Selector="$Config/IntervalSeconds$" ParameterType="int"/>
<OverrideableParameter ID="SyncTime" Selector="$Config/SyncTime$" ParameterType="string"/>
<OverrideableParameter ID="TimeoutSeconds" Selector="$Config/TimeoutSeconds$" ParameterType="int"/>
</OverrideableParameters>
<ModuleImplementation>
<Composite>
<MemberModules>
<DataSource ID="DS" TypeID="Ambari.SCOM.Module.ClusterServiceMetricsDataSource">
<IntervalSeconds>$Config/IntervalSeconds$</IntervalSeconds>
<SyncTime>$Config/SyncTime$</SyncTime>
<TimeoutSeconds>$Config/TimeoutSeconds$</TimeoutSeconds>
<CollectionScript>## Licensed to the Apache Software Foundation (ASF) under one
## or more contributor license agreements. See the NOTICE file
## distributed with this work for additional information
## regarding copyright ownership. The ASF licenses this file
## to you under the Apache License, Version 2.0 (the
## "License"); you may not use this file except in compliance
## with the License. You may obtain a copy of the License at
##
## http://www.apache.org/licenses/LICENSE-2.0
##
## Unless required by applicable law or agreed to in writing,
## software distributed under the License is distributed on an
## "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
## KIND, either express or implied. See the License for the
## specific language governing permissions and limitations
## under the License.

Param ($MonitoringAmbariUri, $Username, $Password)

function Main {
$response = InvokeRestAPI (JoinUri "$MonitoringAmbariUri" 'components/NAMENODE?fields=metrics/dfs') $Username $Password

$propertyBag = $ScriptApi.CreatePropertyBag()

if ($response.metrics -ne $null) {
if (($namenodeGroup = TryGetChildValue $response.metrics @('dfs', 'namenode')) -ne $null) {
if ($namenodeGroup['LiveNodes']) {
try {
$liveNodes = (ParseJsonString $namenodeGroup['LiveNodes']).Count
$propertyBag.AddValue('livenodes', $liveNodes)
} catch {}

try {
$deadNodes = (ParseJsonString $namenodeGroup['DeadNodes']).Count
$propertyBag.AddValue('deadnodes', $deadNodes)
} catch {}

if ($liveNodes -ne $null -and $deadNodes -ne $null -and (($totalNodes = $liveNodes + $deadNodes) -gt 0)) {
$propertyBag.AddValue('deadnodes_percent', [Math]::Round($deadNodes / $totalNodes * 100, 2))
}

try {
$decomNodes = (ParseJsonString $namenodeGroup['DecomNodes']).Count
$propertyBag.AddValue('decomnodes', $decomNodes)
} catch {}
}

$null = AddToBagIfExists $propertyBag 'filescreated' $namenodeGroup 'FilesCreated'
$null = AddToBagIfExists $propertyBag 'filesappended' $namenodeGroup 'FilesAppended'
$null = AddToBagIfExists $propertyBag 'filesdeleted' $namenodeGroup 'FilesDeleted'
}

if (($fsNamesystemGroup = TryGetChildValue $response.metrics @('dfs', 'FSNamesystem')) -ne $null) {
$null = AddToBagIfExists $propertyBag 'filestotal' $fsNamesystemGroup 'FilesTotal'
$totalBlocks = AddToBagIfExists $propertyBag 'blockstotal' $fsNamesystemGroup 'BlocksTotal'
$capacityTotal = AddToBagIfExists $propertyBag 'capacitytotalgb' $fsNamesystemGroup 'CapacityTotalGB'
$capacityUsed = AddToBagIfExists $propertyBag 'capacityusedgb' $fsNamesystemGroup 'CapacityUsedGB'
$capacityRemaining = AddToBagIfExists $propertyBag 'capacityremaininggb' $fsNamesystemGroup 'CapacityRemainingGB'
if ($capacityTotal -ne $null -and $capacityUsed -ne $null -and $capacityRemaining -ne $null -and $capacityTotal -gt 0) {
$propertyBag.AddValue('capacitynondfsusedgb', $capacityTotal - $capacityUsed - $capacityRemaining)
$propertyBag.AddValue('capacityremaining_percent', [Math]::Round($capacityRemaining / $capacityTotal * 100, 2))
}
$null = AddToBagIfExists $propertyBag 'corruptblocks' $fsNamesystemGroup 'CorruptBlocks'
$null = AddToBagIfExists $propertyBag 'pendingdeletionblocks' $fsNamesystemGroup 'PendingDeletionBlocks'
$null = AddToBagIfExists $propertyBag 'pendingreplicationblocks' $fsNamesystemGroup 'PendingReplicationBlocks'
$underReplicatedBlocks = AddToBagIfExists $propertyBag 'underreplicatedblocks' $fsNamesystemGroup 'UnderReplicatedBlocks'
if ($underReplicatedBlocks -ne $null -and $totalBlocks -ne $null -and $totalBlocks -gt 0) {
$propertyBag.AddValue('underreplicatedblocks_percent', [Math]::Round($underReplicatedBlocks / $totalBlocks * 100, 2))
}
$null = AddToBagIfExists $propertyBag 'missingblocks' $fsNamesystemGroup 'MissingBlocks'
}
}

$startTimeStamp = ToUnixEpochTimeStamp([DateTime]::UtcNow.AddMinutes(-10))
$response = InvokeRestAPI (JoinUri "$MonitoringAmbariUri" "components/DATANODE?fields=metrics/dfs/datanode[$startTimeStamp]") $Username $Password

if (($datanodeGroup = TryGetChildValue $response @('metrics', 'dfs', 'datanode')) -ne $null) {
foreach ($metric in $datanodeGroup.Keys) {
$null = AddToBagIfExists $propertyBag ("aggregated.datanode.$metric".ToLower()) (GetLatestValue $datanodeGroup[$metric])
}
}

$propertyBag
}

function GetLatestValue($measures) {
$maxDate = 0
$maxValue = $null
foreach ($measure in $measures) {
if ($measure[1] -le $maxDate) { continue }
$maxDate = $measure[1]
$maxValue = $measure[0]
}
$maxValue
}
</CollectionScript>
<ServiceName>Hdfs</ServiceName>
</DataSource>
</MemberModules>
<Composition>
<Node ID="DS"/>
</Composition>
</Composite>
</ModuleImplementation>
<OutputType>System!System.PropertyBagData</OutputType>
</DataSourceModuleType>