[HAWQ-842] Failed to acquire resource from resource manager - ASF JIRA

XML

Word

Printable

JSON

Details

Type: Bug
Status: Closed
Priority: Major
Resolution: Fixed
Affects Version/s: None
Fix Version/s: 2.0.0.0-incubating
Component/s: Resource Manager
Labels:
None

Description

This is the pg_log:
2016-06-20 17:56:03.864644 CST,,,p526096,th503294176,,,,0,,,seg-10000,,,,,"LOG","00000","database system was shut down at 2016-06-20 17:54:32 CST",,,,,,,0,,"xlog.c",6205,
2016-06-20 17:56:03.864908 CST,,,p526096,th503294176,,,,0,,,seg-10000,,,,,"LOG","00000","checkpoint record is at 0/2672EF8",,,,,,,0,,"xlog.c",6304,
2016-06-20 17:56:03.864923 CST,,,p526096,th503294176,,,,0,,,seg-10000,,,,,"LOG","00000","redo record is at 0/2672EF8; undo record is at 0/0; shutdown TRUE",,,,,,,0,,"xlog.c",6338,
2016-06-20 17:56:03.864933 CST,,,p526096,th503294176,,,,0,,,seg-10000,,,,,"LOG","00000","next transaction ID: 0/1284; next OID: 16514",,,,,,,0,,"xlog.c",6342,
2016-06-20 17:56:03.864942 CST,,,p526096,th503294176,,,,0,,,seg-10000,,,,,"LOG","00000","next MultiXactId: 1; next MultiXactOffset: 0",,,,,,,0,,"xlog.c",6345,
2016-06-20 17:56:03.864951 CST,,,p526096,th503294176,,,,0,,,seg-10000,,,,,"LOG","00000","end of transaction log location is 0/2672F48",,,,,,,0,,"xlog.c",6582,
2016-06-20 17:56:03.865750 CST,,,p526096,th503294176,,,,0,,,seg-10000,,,,,"LOG","00000","Oldest active transaction from prepared transactions 1284",,,,,,,0,,"xlog.c",5996,
2016-06-20 17:56:03.867372 CST,,,p526096,th503294176,,,,0,,,seg-10000,,,,,"LOG","00000","database system is ready",,,,,,,0,,"xlog.c",6022,
2016-06-20 17:56:03.867394 CST,,,p526096,th503294176,,,,0,,,seg-10000,,,,,"LOG","00000","PostgreSQL 8.2.15 (Greenplum Database 4.2.0 build 1) (HAWQ 2.0.0.0 build dev) on x86_64-unknown-linux-gnu, compiled by GCC gcc (GCC) 4.8.0 compiled on Jun 19 2016 03:02:01",,,,,,,0,,"xlog.c",6032,
2016-06-20 17:56:03.868503 CST,,,p526096,th503294176,,,,0,,,seg-10000,,,,,"LOG","00000","Finished normal startup for clean shutdown case",,,,,,,0,,"xlog.c",6810,
2016-06-20 17:56:03.876213 CST,,,p526097,th503294176,,,,0,,,seg-10000,,,,,"LOG","00000","Finished startup integrity checking",,,,,,,0,,"xlog.c",7159,
2016-06-20 17:56:03.879998 CST,,,p526104,th503294176,,,,0,,,seg-10000,,,,,"LOG","00000","HAWQ Segment RM :: Temporary directory /data1/hawq/tmp",,,,,,,0,,"resourcemanager.c",1055,
2016-06-20 17:56:03.880039 CST,,,p526104,th503294176,,,,0,,,seg-10000,,,,,"LOG","00000","checkAndBuildFailedTmpDirList finished checking temporary directory, which costs 41 us",,,,,,,0,,"resourcemanager_RMSEG.c",274,
2016-06-20 17:56:03.883958 CST,,,p526104,th503294176,,,,0,con4,,seg-10000,,,,,"LOG","00000","YARN mode resource broker created resource broker process PID=526105.",,,,,,,0,,"resourcebroker_LIBYARN.c",158,
2016-06-20 17:56:03.884155 CST,,,p526105,th503294176,,,,0,con4,,seg-10000,,,,,"LOG","00000","YARN mode resource broker accepted YARN connection arguments : YARN Server RM_IP_XXXXXX:8032 Scheduler server RM_IP_XXXXXX:8030 Queue hawq Application name hawq, by user:postgres",,,,,,,0,,"resourcebroker_LIBYARN_proc.c",501,
2016-06-20 17:56:03.884283 CST,,,p526104,th503294176,,,,0,con4,,seg-10000,,,,,"LOG","00000","Resource manager starts accepting resource request. Listening normal socket port 5437. Total listened 1 FDs.",,,,,,,0,,"resourcemanager.c",2492,
2016-06-20 17:56:03.884378 CST,,,p526094,th503294176,,,,0,,,seg-10000,,,,,"LOG","00000","Wait for HAWQ RM -1",,,,,,,0,,"resourcemanager.c",421,
2016-06-20 17:56:03.884409 CST,,,p526094,th503294176,,,,0,,,seg-10000,,,,,"LOG","00000","HAWQ :: Received signal notification that HAWQ RM works now.",,,,,,,0,,"resourcemanager.c",429,
2016-06-20 17:56:03.884424 CST,,,p526094,th503294176,,,,0,,,seg-10000,,,,,"LOG","00000","PostgreSQL 8.2.15 (Greenplum Database 4.2.0 build 1) (HAWQ 2.0.0.0 build dev) on x86_64-unknown-linux-gnu, compiled by GCC gcc (GCC) 4.8.0 compiled on Jun 19 2016 03:02:03",,,,,,,0,,"postmaster.c",3694,
2016-06-20 17:56:03.884441 CST,,,p526094,th503294176,,,,0,,,seg-10000,,,,,"LOG","00000","database system is ready to accept connections","PostgreSQL 8.2.15 (Greenplum Database 4.2.0 build 1) (HAWQ 2.0.0.0 build dev) on x86_64-unknown-linux-gnu, compiled by GCC gcc (GCC) 4.8.0 compiled on Jun 19 2016 03:02:03",,,,,,0,,"postmaster.c",3701,
2016-06-20 17:56:03.885673 CST,,,p526095,th503294176,,,,0,,,seg-10000,,,,,"LOG","00000","3rd party error log:
2016-06-20 17:56:03.885486, p526105, th140510358382816, INFO ApplicationClient session auth method : simple",,,,,,,,"SysLoggerMain","syslogger.c",518,
2016-06-20 17:56:03.892215 CST,,,p526104,th503294176,,,,0,con4,,seg-10000,,,,,"LOG","00000","Cleanup segment configuration catalog table successfully!",,,,,,,0,,"resourcepool.c",460,
2016-06-20 17:56:03.892604 CST,,,p526095,th503294176,,,,0,,,seg-10000,,,,,"LOG","00000","3rd party error log:
2016-06-20 17:56:03.892561, p526105, th140510358382816, INFO Yarn RM HA is not configured.",,,,,,,,"SysLoggerMain","syslogger.c",518,
2016-06-20 17:56:03.892643 CST,,,p526095,th503294176,,,,0,,,seg-10000,,,,,"LOG","00000","3rd party error log:
2016-06-20 17:56:03.892600, p526105, th140510358382816, INFO ApplicationClient Resource Manager HA is disable.",,,,,,,,"SysLoggerMain","syslogger.c",518,
2016-06-20 17:56:03.893081 CST,,,p526095,th503294176,,,,0,,,seg-10000,,,,,"LOG","00000","3rd party error log:
2016-06-20 17:56:03.893051, p526105, th140510358382816, INFO ApplicationClient::getActiveAppClientProto is called.",,,,,,,,"SysLoggerMain","syslogger.c",518,
2016-06-20 17:56:03.893116 CST,,,p526095,th503294176,,,,0,,,seg-10000,,,,,"LOG","00000","3rd party error log:
2016-06-20 17:56:03.893068, p526105, th140510358382816, INFO ApplicationClient::getActiveAppClientProto, current is 0.",,,,,,,,"SysLoggerMain","syslogger.c",518,
2016-06-20 17:56:03.895989 CST,,,p526095,th503294176,,,,0,,,seg-10000,,,,,"LOG","00000","3rd party error log:
2016-06-20 17:56:03.895940, p526105, th140510358382816, INFO LibYarnClient::createJob, getNewApplication finished, appId:[clusterTimeStamp:1466415720399,id:2]",,,,,,,,"SysLoggerMain","syslogger.c",518,
2016-06-20 17:56:03.896107 CST,,,p526095,th503294176,,,,0,,,seg-10000,,,,,"LOG","00000","3rd party error log:
2016-06-20 17:56:03.896075, p526105, th140510358382816, INFO ApplicationClient::getActiveAppClientProto is called.",,,,,,,,"SysLoggerMain","syslogger.c",518,
2016-06-20 17:56:03.896151 CST,,,p526095,th503294176,,,,0,,,seg-10000,,,,,"LOG","00000","3rd party error log:
2016-06-20 17:56:03.896094, p526105, th140510358382816, INFO ApplicationClient::getActiveAppClientProto, current is 0.",,,,,,,,"SysLoggerMain","syslogger.c",518,
2016-06-20 17:56:03.896602 CST,,,p526104,th503294176,,,,0,con4,,seg-10000,,,,,"LOG","00000","Cleanup segment configuration history catalog table successfully, keep period: recent 365 days.",,,,,,,0,,"resourcepool.c",530,
2016-06-20 17:56:03.897807 CST,,,p526095,th503294176,,,,0,,,seg-10000,,,,,"LOG","00000","3rd party error log:
2016-06-20 17:56:03.897769, p526105, th140510358382816, INFO LibYarnClient::createJob, submitApplication finished",,,,,,,,"SysLoggerMain","syslogger.c",518,
2016-06-20 17:56:03.897862 CST,,,p526095,th503294176,,,,0,,,seg-10000,,,,,"LOG","00000","3rd party error log:
2016-06-20 17:56:03.897833, p526105, th140510358382816, INFO ApplicationClient::getActiveAppClientProto is called.",,,,,,,,"SysLoggerMain","syslogger.c",518,
2016-06-20 17:56:03.897898 CST,,,p526095,th503294176,,,,0,,,seg-10000,,,,,"LOG","00000","3rd party error log:
2016-06-20 17:56:03.897851, p526105, th140510358382816, INFO ApplicationClient::getActiveAppClientProto, current is 0.",,,,,,,,"SysLoggerMain","syslogger.c",518,
2016-06-20 17:56:03.901293 CST,,,p526104,th503294176,,,,0,con4,,seg-10000,,,,,"LOG","00000","Add a new row into segment configuration catalog table,registration order:0, role:m, status:u, port:5432, hostname:Master_ip, address:Master_ip, description:",,,,,,,0,,"resourcepool.c",879,
2016-06-20 17:56:03.901503 CST,,,p526095,th503294176,,,,0,,,seg-10000,,,,,"LOG","00000","3rd party error log:
2016-06-20 17:56:03.901443, p526105, th140510358382816, INFO LibYarnClient::createJob, appId[cluster_timestamp:1466415720399,id:2], appState:4",,,,,,,,"SysLoggerMain","syslogger.c",518,
2016-06-20 17:56:03.901535 CST,,,p526095,th503294176,,,,0,,,seg-10000,,,,,"LOG","00000","3rd party error log:
2016-06-20 17:56:03.901492, p526105, th140510358382816, INFO ApplicationClient::getActiveAppClientProto is called.
2016-06-20 17:56:03.901502, p526105, th140510358382816, INFO ApplicationClient::getActiveAppClientProto, current is 0.",,,,,,,,"SysLoggerMain","syslogger.c",518,
2016-06-20 17:56:03.905203 CST,,,p526104,th503294176,,,,0,con4,,seg-10000,,,,,"LOG","00000","Resource manager successfully loaded role specifications.",,,,,,,0,,"resourcemanager.c",1272,
2016-06-20 17:56:03.907677 CST,,,p526095,th503294176,,,,0,,,seg-10000,,,,,"LOG","00000","3rd party error log:

hawq-site.xml:
<configuration>
<property>
<name>hawq_master_address_host</name>
<value>HAWQ_Master</value>
<description>The host name of hawq master.</description>
</property>

<property>
<name>hawq_master_address_port</name>
<value>5432</value>
<description>The port of hawq master.</description>
</property>

<property>
<name>hawq_standby_address_host</name>
<value>none</value>
<description>The host name of hawq standby master.</description>
</property>

<property>
<name>hawq_segment_address_port</name>
<value>40000</value>
<description>The port of hawq segment.</description>
</property>

<property>
<name>hawq_dfs_url</name>
<value>thallo/hawq_default</value>
<description>URL for accessing HDFS.</description>
</property>

<property>
<name>hawq_master_directory</name>
<value>/data1/hawq-data-directory/masterdd</value>
<description>The directory of hawq master.</description>
</property>

<property>
<name>hawq_segment_directory</name>
<value>/data1/hawq-data-directory/segmentdd</value>
<description>The directory of hawq segment.</description>
</property>

<property>
<name>hawq_master_temp_directory</name>
<value>/data1/hawq/tmp</value>
<description>The temporary directory reserved for hawq master.</description>
</property>

<property>
<name>hawq_segment_temp_directory</name>
<value>/data1/hawq/tmp</value>
<description>The temporary directory reserved for hawq segment.</description>
</property>

<property>
<name>hawq_global_rm_type</name>
<value>yarn</value>
<description>The resource manager type to start for allocating resource.
'none' means hawq resource manager exclusively uses whole
cluster; 'yarn' means hawq resource manager contacts YARN
resource manager to negotiate resource.
</description>
</property>

<property>
<name>hawq_rm_memory_limit_perseg</name>
<value>64GB</value>
<description>The limit of memory usage in a hawq segment when
hawq_global_rm_type is set 'none'.
</description>
</property>

<property>
<name>hawq_rm_nvcore_limit_perseg</name>
<value>16</value>
<description>The limit of virtual core usage in a hawq segment when
hawq_global_rm_type is set 'none'.
</description>
</property>

<property>
<name>hawq_rm_yarn_address</name>
<value>RM_IP:8032</value>
<description>The address of YARN resource manager server.</description>
</property>

<property>
<name>hawq_rm_yarn_scheduler_address</name>
<value>RM_IP:8030</value>
<description>The address of YARN scheduler server.</description>
</property>

<property>
<name>hawq_rm_yarn_queue_name</name>
<value>hawq</value>
<description>The YARN queue name to register hawq resource manager.</description>
</property>

<property>
<name>hawq_rm_yarn_app_name</name>
<value>hawq</value>
<description>The application name to register hawq resource manager in YARN.</description>
</property>

<property>
<name>hawq_re_cpu_enable</name>
<value>false</value>
<description>The control to enable/disable CPU resource enforcement.</description>
</property>

<property>
<name>hawq_re_cgroup_mount_point</name>
<value>/sys/fs/cgroup</value>
<description>The mount point of CGroup file system for resource enforcement.
For example, /sys/fs/cgroup/cpu/hawq for CPU sub-system.
</description>
</property>

<property>
<name>hawq_re_cgroup_hierarchy_name</name>
<value>hawq</value>
<description>The name of the hierarchy to accomodate CGroup directories/files for resource enforcement.
For example, /sys/fs/cgroup/cpu/hawq for CPU sub-system.
</description>
</property>

<property>
<name>default_hash_table_bucket_number</name>
<value>6</value>
</property>
</configuration>

Attachments

Activity

People

Assignee:: Lei Chang

Reporter:: Biao Wu

Votes:: 0 Vote for this issue

Watchers:: 4 Start watching this issue

Dates

Created:: 21/Jun/16 01:32

Updated:: 14/Jul/16 18:23

Resolved:: 14/Jul/16 18:23